diff --git a/.clang-tidy b/.clang-tidy index 13c1b116ead..dc1cebe9430 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -96,7 +96,6 @@ Checks: [ '-modernize-use-default-member-init', '-modernize-use-emplace', '-modernize-use-nodiscard', - '-modernize-use-override', '-modernize-use-trailing-return-type', '-performance-inefficient-string-concatenation', @@ -120,7 +119,6 @@ Checks: [ '-readability-named-parameter', '-readability-redundant-declaration', '-readability-simplify-boolean-expr', - '-readability-static-accessed-through-instance', '-readability-suspicious-call-argument', '-readability-uppercase-literal-suffix', '-readability-use-anyofallof', diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7fb2abebbbb..85b1d460833 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -40,3 +40,45 @@ At a minimum, the following information should be added (but add more as needed) > Information about CI checks: https://clickhouse.com/docs/en/development/continuous-integration/ + +--- +### Modify your CI run: +**NOTE:** If your merge the PR with modified CI you **MUST KNOW** what you are doing +**NOTE:** Checked options will be applied if set before CI RunConfig/PrepareRunConfig step + +#### Include tests (required builds will be added automatically): +- [ ] Fast test +- [ ] Integration Tests +- [ ] Stateless tests +- [ ] Stateful tests +- [ ] Unit tests +- [ ] Performance tests +- [ ] All with ASAN +- [ ] All with TSAN +- [ ] All with Analyzer +- [ ] Add your option here + +#### Exclude tests: +- [ ] Fast test +- [ ] Integration Tests +- [ ] Stateless tests +- [ ] Stateful tests +- [ ] Performance tests +- [ ] All with ASAN +- [ ] All with TSAN +- [ ] All with MSAN +- [ ] All with UBSAN +- [ ] All with Coverage +- [ ] All with Aarch64 +- [ ] Add your option here + +#### Extra options: +- [ ] do not test (only style check) +- [ ] disable merge-commit (no merge from master before tests) +- [ ] disable CI cache (job reuse) + +#### Only specified batches in multi-batch jobs: +- [ ] 1 +- [ ] 2 +- [ ] 3 +- [ ] 4 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2853adff48a..816bdfd4f31 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -374,7 +374,7 @@ jobs: if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml with: - test_name: Stateless tests (release, analyzer, s3, DatabaseReplicated) + test_name: Stateless tests (release, old analyzer, s3, DatabaseReplicated) runner_type: func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestS3Debug: @@ -632,7 +632,7 @@ jobs: if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml with: - test_name: Integration tests (asan, analyzer) + test_name: Integration tests (asan, old analyzer) runner_type: stress-tester data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2dddde9aa14..74ce8452de8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -6,6 +6,7 @@ env: PYTHONUNBUFFERED: 1 on: # yamllint disable-line rule:truthy + merge_group: pull_request: types: - synchronize @@ -29,6 +30,7 @@ jobs: fetch-depth: 0 # to get version filter: tree:0 - name: Labels check + if: ${{ github.event_name != 'merge_group' }} run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 run_check.py @@ -56,16 +58,9 @@ jobs: echo 'EOF' } >> "$GITHUB_OUTPUT" - name: Re-create GH statuses for skipped jobs if any + if: ${{ github.event_name != 'merge_group' }} run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses - - name: Style check early - # hack to run style check before the docker build job if possible (style-check image not changed) - if: contains(fromJson(steps.runconfig.outputs.CI_DATA).jobs_data.jobs_to_do, 'Style check early') - run: | - DOCKER_TAG=$(echo '${{ toJson(fromJson(steps.runconfig.outputs.CI_DATA).docker_data.images) }}' | tr -d '\n') - export DOCKER_TAG=$DOCKER_TAG - python3 ./tests/ci/style_check.py --no-push - python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --post --job-name 'Style check' BuildDockers: needs: [RunConfig] if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }} @@ -162,7 +157,7 @@ jobs: ################################# Stage Final ################################# # FinishCheck: - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && github.event_name != 'merge_group' }} needs: [Tests_1, Tests_2] runs-on: [self-hosted, style-checker] steps: diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 9e95b3d3d8f..4d45c8d8d4b 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -436,7 +436,7 @@ jobs: if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml with: - test_name: Integration tests (asan, analyzer) + test_name: Integration tests (asan, old analyzer) runner_type: stress-tester data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: diff --git a/.gitignore b/.gitignore index 1ea8f83dcc2..db3f77d7d1e 100644 --- a/.gitignore +++ b/.gitignore @@ -164,6 +164,9 @@ tests/queries/0_stateless/*.generated-expect tests/queries/0_stateless/*.expect.history tests/integration/**/_gen +# pytest --pdb history +.pdb_history + # rust /rust/**/target* # It is autogenerated from *.in diff --git a/.gitmessage b/.gitmessage index 2ad30596de6..797446edd49 100644 --- a/.gitmessage +++ b/.gitmessage @@ -26,4 +26,4 @@ ## To run only specified batches for multi-batch job(s) #batch_2 -#btach_1_2_3 +#batch_1_2_3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9df678d4b9a..dd88f3ee2c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,183 @@ ### Table of Contents +**[ClickHouse release v24.3 LTS, 2024-03-26](#243)**
**[ClickHouse release v24.2, 2024-02-29](#242)**
**[ClickHouse release v24.1, 2024-01-30](#241)**
**[Changelog for 2023](https://clickhouse.com/docs/en/whats-new/changelog/2023/)**
# 2024 Changelog +### ClickHouse release 24.3 LTS, 2024-03-27 + +#### Upgrade Notes +* The setting `allow_experimental_analyzer` is enabled by default and it switches the query analysis to a new implementation, which has better compatibility and feature completeness. The feature "analyzer" is considered beta instead of experimental. You can turn the old behavior by setting the `compatibility` to `24.2` or disabling the `allow_experimental_analyzer` setting. Watch the [video on YouTube](https://www.youtube.com/watch?v=zhrOYQpgvkk). +* ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. This is controlled by the settings, `output_format_parquet_string_as_string`, `output_format_orc_string_as_string`, `output_format_arrow_string_as_string`. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases. Parquet/ORC/Arrow supports many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools lack support for the faster `lz4` compression method, that's why we set `zstd` by default. This is controlled by the settings `output_format_parquet_compression_method`, `output_format_orc_compression_method`, and `output_format_arrow_compression_method`. We changed the default to `zstd` for Parquet and ORC, but not Arrow (it is emphasized for low-level usages). [#61817](https://github.com/ClickHouse/ClickHouse/pull/61817) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In the new ClickHouse version, the functions `geoDistance`, `greatCircleDistance`, and `greatCircleAngle` will use 64-bit double precision floating point data type for internal calculations and return type if all the arguments are Float64. This closes [#58476](https://github.com/ClickHouse/ClickHouse/issues/58476). In previous versions, the function always used Float32. You can switch to the old behavior by setting `geo_distance_returns_float64_on_float64_arguments` to `false` or setting `compatibility` to `24.2` or earlier. [#61848](https://github.com/ClickHouse/ClickHouse/pull/61848) ([Alexey Milovidov](https://github.com/alexey-milovidov)). Co-authored with [Geet Patel](https://github.com/geetptl). +* The obsolete in-memory data parts have been deprecated since version 23.5 and have not been supported since version 23.10. Now the remaining code is removed. Continuation of [#55186](https://github.com/ClickHouse/ClickHouse/issues/55186) and [#45409](https://github.com/ClickHouse/ClickHouse/issues/45409). It is unlikely that you have used in-memory data parts because they were available only before version 23.5 and only when you enabled them manually by specifying the corresponding SETTINGS for a MergeTree table. To check if you have in-memory data parts, run the following query: `SELECT part_type, count() FROM system.parts GROUP BY part_type ORDER BY part_type`. To disable the usage of in-memory data parts, do `ALTER TABLE ... MODIFY SETTING min_bytes_for_compact_part = DEFAULT, min_rows_for_compact_part = DEFAULT`. Before upgrading from old ClickHouse releases, first check that you don't have in-memory data parts. If there are in-memory data parts, disable them first, then wait while there are no in-memory data parts and continue the upgrade. [#61127](https://github.com/ClickHouse/ClickHouse/pull/61127) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Changed the column name from `duration_ms` to `duration_microseconds` in the `system.zookeeper` table to reflect the reality that the duration is in the microsecond resolution. [#60774](https://github.com/ClickHouse/ClickHouse/pull/60774) ([Duc Canh Le](https://github.com/canhld94)). +* Reject incoming INSERT queries in case when query-level settings `async_insert` and `deduplicate_blocks_in_dependent_materialized_views` are enabled together. This behaviour is controlled by a setting `throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert` and enabled by default. This is a continuation of https://github.com/ClickHouse/ClickHouse/pull/59699 needed to unblock https://github.com/ClickHouse/ClickHouse/pull/59915. [#60888](https://github.com/ClickHouse/ClickHouse/pull/60888) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Utility `clickhouse-copier` is moved to a separate repository on GitHub: https://github.com/ClickHouse/copier. It is no longer included in the bundle but is still available as a separate download. This closes: [#60734](https://github.com/ClickHouse/ClickHouse/issues/60734) This closes: [#60540](https://github.com/ClickHouse/ClickHouse/issues/60540) This closes: [#60250](https://github.com/ClickHouse/ClickHouse/issues/60250) This closes: [#52917](https://github.com/ClickHouse/ClickHouse/issues/52917) This closes: [#51140](https://github.com/ClickHouse/ClickHouse/issues/51140) This closes: [#47517](https://github.com/ClickHouse/ClickHouse/issues/47517) This closes: [#47189](https://github.com/ClickHouse/ClickHouse/issues/47189) This closes: [#46598](https://github.com/ClickHouse/ClickHouse/issues/46598) This closes: [#40257](https://github.com/ClickHouse/ClickHouse/issues/40257) This closes: [#36504](https://github.com/ClickHouse/ClickHouse/issues/36504) This closes: [#35485](https://github.com/ClickHouse/ClickHouse/issues/35485) This closes: [#33702](https://github.com/ClickHouse/ClickHouse/issues/33702) This closes: [#26702](https://github.com/ClickHouse/ClickHouse/issues/26702). +* To increase compatibility with MySQL, the compatibility alias `locate` now accepts arguments `(needle, haystack[, start_pos])` by default. The previous behavior `(haystack, needle, [, start_pos])` can be restored by setting `function_locate_has_mysql_compatible_argument_order = 0`. [#61092](https://github.com/ClickHouse/ClickHouse/pull/61092) ([Robert Schulze](https://github.com/rschu1ze)). +* Forbid `SimpleAggregateFunction` in `ORDER BY` of `MergeTree` tables (like `AggregateFunction` is forbidden, but they are forbidden because they are not comparable) by default (use `allow_suspicious_primary_key` to allow them). [#61399](https://github.com/ClickHouse/ClickHouse/pull/61399) ([Azat Khuzhin](https://github.com/azat)). +* The `Ordinary` database engine is deprecated. You will receive a warning in clickhouse-client if your server is using it. This closes [#52229](https://github.com/ClickHouse/ClickHouse/issues/52229). [#56942](https://github.com/ClickHouse/ClickHouse/pull/56942) ([shabroo](https://github.com/shabroo)). + +#### New Feature +* Support reading and writing backups as `tar` (in addition to `zip`). [#59535](https://github.com/ClickHouse/ClickHouse/pull/59535) ([josh-hildred](https://github.com/josh-hildred)). +* Implemented support for S3 Express buckets. [#59965](https://github.com/ClickHouse/ClickHouse/pull/59965) ([Nikita Taranov](https://github.com/nickitat)). +* Allow to attach parts from a different disk (using copy instead of hard link). [#60112](https://github.com/ClickHouse/ClickHouse/pull/60112) ([Unalian](https://github.com/Unalian)). +* Size-capped `Memory` tables: controlled by their settings, `min_bytes_to_keep, max_bytes_to_keep, min_rows_to_keep` and `max_rows_to_keep`. [#60612](https://github.com/ClickHouse/ClickHouse/pull/60612) ([Jake Bamrah](https://github.com/JakeBamrah)). +* Separate limits on number of waiting and executing queries. Added new server setting `max_waiting_queries` that limits the number of queries waiting due to `async_load_databases`. Existing limits on number of executing queries no longer count waiting queries. [#61053](https://github.com/ClickHouse/ClickHouse/pull/61053) ([Sergei Trifonov](https://github.com/serxa)). +* Added a table `system.keywords` which contains all the keywords from parser. Mostly needed and will be used for better fuzzing and syntax highlighting. [#51808](https://github.com/ClickHouse/ClickHouse/pull/51808) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add support for `ATTACH PARTITION ALL`. [#61107](https://github.com/ClickHouse/ClickHouse/pull/61107) ([Kirill Nikiforov](https://github.com/allmazz)). +* Add a new function, `getClientHTTPHeader`. This closes [#54665](https://github.com/ClickHouse/ClickHouse/issues/54665). Co-authored with @lingtaolf. [#61820](https://github.com/ClickHouse/ClickHouse/pull/61820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `generate_series` as a table function (compatibility alias for PostgreSQL to the existing `numbers` function). This function generates table with an arithmetic progression with natural numbers. [#59390](https://github.com/ClickHouse/ClickHouse/pull/59390) ([divanik](https://github.com/divanik)). +* A mode for `topK`/`topkWeighed` support mode, which return count of values and its error. [#54508](https://github.com/ClickHouse/ClickHouse/pull/54508) ([UnamedRus](https://github.com/UnamedRus)). +* Added function `toMillisecond` which returns the millisecond component for values of type`DateTime` or `DateTime64`. [#60281](https://github.com/ClickHouse/ClickHouse/pull/60281) ([Shaun Struwig](https://github.com/Blargian)). +* Allow configuring HTTP redirect handlers for clickhouse-server. For example, you can make `/` redirect to the Play UI. [#60390](https://github.com/ClickHouse/ClickHouse/pull/60390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Performance Improvement +* Optimized function `dotProduct` to omit unnecessary and expensive memory copies. [#60928](https://github.com/ClickHouse/ClickHouse/pull/60928) ([Robert Schulze](https://github.com/rschu1ze)). +* 30x faster printing for 256-bit integers. [#61100](https://github.com/ClickHouse/ClickHouse/pull/61100) ([Raúl Marín](https://github.com/Algunenano)). +* If the table's primary key contains mostly useless columns, don't keep them in memory. This is controlled by a new setting `primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns` with the value `0.9` by default, which means: for a composite primary key, if a column changes its value for at least 0.9 of all the times, the next columns after it will be not loaded. [#60255](https://github.com/ClickHouse/ClickHouse/pull/60255) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve the performance of serialized aggregation method when involving multiple `Nullable` columns. [#55809](https://github.com/ClickHouse/ClickHouse/pull/55809) ([Amos Bird](https://github.com/amosbird)). +* Lazy build JSON's output to improve performance of ALL JOIN. [#58278](https://github.com/ClickHouse/ClickHouse/pull/58278) ([LiuNeng](https://github.com/liuneng1994)). +* Make HTTP/HTTPs connections with external services, such as AWS S3 reusable for all uses cases. Even when response is 3xx or 4xx. [#58845](https://github.com/ClickHouse/ClickHouse/pull/58845) ([Sema Checherinda](https://github.com/CheSema)). +* Improvements to aggregate functions `argMin` / `argMax` / `any` / `anyLast` / `anyHeavy`, as well as `ORDER BY {u8/u16/u32/u64/i8/i16/u32/i64) LIMIT 1` queries. [#58640](https://github.com/ClickHouse/ClickHouse/pull/58640) ([Raúl Marín](https://github.com/Algunenano)). +* Trivial optimization for column's filter. Peak memory can be reduced to 44% of the original in some cases. [#59698](https://github.com/ClickHouse/ClickHouse/pull/59698) ([李扬](https://github.com/taiyang-li)). +* Execute `multiIf` function in a columnar fashion when the result type's underlying type is a number. [#60384](https://github.com/ClickHouse/ClickHouse/pull/60384) ([李扬](https://github.com/taiyang-li)). +* Faster (almost 2x) mutexes. [#60823](https://github.com/ClickHouse/ClickHouse/pull/60823) ([Azat Khuzhin](https://github.com/azat)). +* Drain multiple connections in parallel when a distributed query is finishing. [#60845](https://github.com/ClickHouse/ClickHouse/pull/60845) ([lizhuoyu5](https://github.com/lzydmxy)). +* Optimize data movement between columns of a Nullable number or a Nullable string, which improves some micro-benchmarks. [#60846](https://github.com/ClickHouse/ClickHouse/pull/60846) ([李扬](https://github.com/taiyang-li)). +* Operations with the filesystem cache will suffer less from the lock contention. [#61066](https://github.com/ClickHouse/ClickHouse/pull/61066) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Optimize array join and other JOINs by preventing a wrong compiler's optimization. Close [#61074](https://github.com/ClickHouse/ClickHouse/issues/61074). [#61075](https://github.com/ClickHouse/ClickHouse/pull/61075) ([李扬](https://github.com/taiyang-li)). +* If a query with a syntax error contained `COLUMNS` matcher with a regular expression, the regular expression was compiled each time during the parser's backtracking, instead of being compiled once. This was a fundamental error. The compiled regexp was put to AST. But the letter A in AST means "abstract" which means it should not contain heavyweight objects. Parts of AST can be created and discarded during parsing, including a large number of backtracking. This leads to slowness on the parsing side and consequently allows DoS by a readonly user. But the main problem is that it prevents progress in fuzzers. [#61543](https://github.com/ClickHouse/ClickHouse/pull/61543) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a new analyzer pass to optimize the IN operator for a single value. [#61564](https://github.com/ClickHouse/ClickHouse/pull/61564) ([LiuNeng](https://github.com/liuneng1994)). +* DNSResolver shuffles set of resolved IPs which is needed to uniformly utilize multiple endpoints of AWS S3. [#60965](https://github.com/ClickHouse/ClickHouse/pull/60965) ([Sema Checherinda](https://github.com/CheSema)). + +#### Experimental Feature +* Support parallel reading for Azure blob storage. This improves the performance of the experimental Azure object storage. [#61503](https://github.com/ClickHouse/ClickHouse/pull/61503) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Add asynchronous WriteBuffer for Azure blob storage similar to S3. This improves the performance of the experimental Azure object storage. [#59929](https://github.com/ClickHouse/ClickHouse/pull/59929) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Use managed identity for backups IO when using Azure Blob Storage. Add a setting to prevent ClickHouse from attempting to create a non-existent container, which requires permissions at the storage account level. [#61785](https://github.com/ClickHouse/ClickHouse/pull/61785) ([Daniel Pozo Escalona](https://github.com/danipozo)). +* Add a setting `parallel_replicas_allow_in_with_subquery = 1` which allows subqueries for IN work with parallel replicas. [#60950](https://github.com/ClickHouse/ClickHouse/pull/60950) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* A change for the "zero-copy" replication: all zero copy locks related to a table have to be dropped when the table is dropped. The directory which contains these locks has to be removed also. [#57575](https://github.com/ClickHouse/ClickHouse/pull/57575) ([Sema Checherinda](https://github.com/CheSema)). + +#### Improvement +* Use `MergeTree` as a default table engine. [#60524](https://github.com/ClickHouse/ClickHouse/pull/60524) ([Alexey Milovidov](https://github.com/alexey-milovidov)) +* Enable `output_format_pretty_row_numbers` by default. It is better for usability. [#61791](https://github.com/ClickHouse/ClickHouse/pull/61791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In the previous version, some numbers in Pretty formats were not pretty enough. [#61794](https://github.com/ClickHouse/ClickHouse/pull/61794) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A long value in Pretty formats won't be cut if it is the single value in the resultset, such as in the result of the `SHOW CREATE TABLE` query. [#61795](https://github.com/ClickHouse/ClickHouse/pull/61795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Similarly to `clickhouse-local`, `clickhouse-client` will accept the `--output-format` option as a synonym to the `--format` option. This closes [#59848](https://github.com/ClickHouse/ClickHouse/issues/59848). [#61797](https://github.com/ClickHouse/ClickHouse/pull/61797) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* If stdout is a terminal and the output format is not specified, `clickhouse-client` and similar tools will use `PrettyCompact` by default, similarly to the interactive mode. `clickhouse-client` and `clickhouse-local` will handle command line arguments for input and output formats in a unified fashion. This closes [#61272](https://github.com/ClickHouse/ClickHouse/issues/61272). [#61800](https://github.com/ClickHouse/ClickHouse/pull/61800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Underscore digit groups in Pretty formats for better readability. This is controlled by a new setting, `output_format_pretty_highlight_digit_groups`. [#61802](https://github.com/ClickHouse/ClickHouse/pull/61802) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add ability to override initial INSERT settings via `SYSTEM FLUSH DISTRIBUTED`. [#61832](https://github.com/ClickHouse/ClickHouse/pull/61832) ([Azat Khuzhin](https://github.com/azat)). +* Enable processors profiling (time spent/in and out bytes for sorting, aggregation, ...) by default. [#61096](https://github.com/ClickHouse/ClickHouse/pull/61096) ([Azat Khuzhin](https://github.com/azat)). +* Support files without format extension in Filesystem database. [#60795](https://github.com/ClickHouse/ClickHouse/pull/60795) ([Kruglov Pavel](https://github.com/Avogar)). +* Make all format names case insensitive, like Tsv, or TSV, or tsv, or even rowbinary. [#60420](https://github.com/ClickHouse/ClickHouse/pull/60420) ([豪肥肥](https://github.com/HowePa)). I appreciate if you will continue to write it correctly, e.g., `JSON` 😇, not `Json` 🤮, but we don't mind if you spell it as you prefer. +* Added `none_only_active` mode for `distributed_ddl_output_mode` setting. [#60340](https://github.com/ClickHouse/ClickHouse/pull/60340) ([Alexander Tokmakov](https://github.com/tavplubix)). +* The advanced dashboard has slightly better colors for multi-line graphs. [#60391](https://github.com/ClickHouse/ClickHouse/pull/60391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The Advanced dashboard now has controls always visible on scrolling. This allows you to add a new chart without scrolling up. [#60692](https://github.com/ClickHouse/ClickHouse/pull/60692) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* While running the `MODIFY COLUMN` query for materialized views, check the inner table's structure to ensure every column exists. [#47427](https://github.com/ClickHouse/ClickHouse/pull/47427) ([sunny](https://github.com/sunny19930321)). +* String types and Enums can be used in the same context, such as: arrays, UNION queries, conditional expressions. This closes [#60726](https://github.com/ClickHouse/ClickHouse/issues/60726). [#60727](https://github.com/ClickHouse/ClickHouse/pull/60727) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow declaring Enums in the structure of external data for query processing (this is an immediate temporary table that you can provide for your query). [#57857](https://github.com/ClickHouse/ClickHouse/pull/57857) ([Duc Canh Le](https://github.com/canhld94)). +* Consider lightweight deleted rows when selecting parts to merge, so the disk size of the resulting part will be estimated better. [#58223](https://github.com/ClickHouse/ClickHouse/pull/58223) ([Zhuo Qiu](https://github.com/jewelzqiu)). +* Added comments for columns for more system tables. Continuation of https://github.com/ClickHouse/ClickHouse/pull/58356. [#59016](https://github.com/ClickHouse/ClickHouse/pull/59016) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Now we can use virtual columns in PREWHERE. It's worthwhile for non-const virtual columns like `_part_offset`. [#59033](https://github.com/ClickHouse/ClickHouse/pull/59033) ([Amos Bird](https://github.com/amosbird)). Improved overall usability of virtual columns. Now it is allowed to use virtual columns in `PREWHERE` (it's worthwhile for non-const virtual columns like `_part_offset`). Now a builtin documentation is available for virtual columns as a comment of column in `DESCRIBE` query with enabled setting `describe_include_virtual_columns`. [#60205](https://github.com/ClickHouse/ClickHouse/pull/60205) ([Anton Popov](https://github.com/CurtizJ)). +* Instead of using a constant key, now object storage generates key for determining remove objects capability. [#59495](https://github.com/ClickHouse/ClickHouse/pull/59495) ([Sema Checherinda](https://github.com/CheSema)). +* Allow "local" as object storage type instead of "local_blob_storage". [#60165](https://github.com/ClickHouse/ClickHouse/pull/60165) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Parallel flush of pending INSERT blocks of Distributed engine on `DETACH`/server shutdown and `SYSTEM FLUSH DISTRIBUTED` (Parallelism will work only if you have multi-disk policy for a table (like everything in the Distributed engine right now)). [#60225](https://github.com/ClickHouse/ClickHouse/pull/60225) ([Azat Khuzhin](https://github.com/azat)). +* Add a setting to force read-through cache for merges. [#60308](https://github.com/ClickHouse/ClickHouse/pull/60308) ([Kseniia Sumarokova](https://github.com/kssenii)). +* An improvement for the MySQL compatibility protocol. The issue [#57598](https://github.com/ClickHouse/ClickHouse/issues/57598) mentions a variant behaviour regarding transaction handling. An issued COMMIT/ROLLBACK when no transaction is active is reported as an error contrary to MySQL behaviour. [#60338](https://github.com/ClickHouse/ClickHouse/pull/60338) ([PapaToemmsn](https://github.com/PapaToemmsn)). +* Function `substring` now has a new alias `byteSlice`. [#60494](https://github.com/ClickHouse/ClickHouse/pull/60494) ([Robert Schulze](https://github.com/rschu1ze)). +* Renamed server setting `dns_cache_max_size` to `dns_cache_max_entries` to reduce ambiguity. [#60500](https://github.com/ClickHouse/ClickHouse/pull/60500) ([Kirill Nikiforov](https://github.com/allmazz)). +* `SHOW INDEX | INDEXES | INDICES | KEYS` no longer sorts by the primary key columns (which was unintuitive). [#60514](https://github.com/ClickHouse/ClickHouse/pull/60514) ([Robert Schulze](https://github.com/rschu1ze)). +* Keeper improvement: abort during startup if an invalid snapshot is detected to avoid data loss. [#60537](https://github.com/ClickHouse/ClickHouse/pull/60537) ([Antonio Andelic](https://github.com/antonio2368)). +* Update tzdata to 2024a. [#60768](https://github.com/ClickHouse/ClickHouse/pull/60768) ([Raúl Marín](https://github.com/Algunenano)). +* Keeper improvement: support `leadership_expiry_ms` in Keeper's settings. [#60806](https://github.com/ClickHouse/ClickHouse/pull/60806) ([Brokenice0415](https://github.com/Brokenice0415)). +* Always infer exponential numbers in JSON formats regardless of the setting `input_format_try_infer_exponent_floats`. Add setting `input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects` that allows to use String type for ambiguous paths instead of an exception during named Tuples inference from JSON objects. [#60808](https://github.com/ClickHouse/ClickHouse/pull/60808) ([Kruglov Pavel](https://github.com/Avogar)). +* Add support for `START TRANSACTION` syntax typically used in MySQL syntax, resolving https://github.com/ClickHouse/ClickHouse/discussions/60865. [#60886](https://github.com/ClickHouse/ClickHouse/pull/60886) ([Zach Naimon](https://github.com/ArctypeZach)). +* Add a flag for the full-sorting merge join algorithm to treat null as biggest/smallest. So the behavior can be compitable with other SQL systems, like Apache Spark. [#60896](https://github.com/ClickHouse/ClickHouse/pull/60896) ([loudongfeng](https://github.com/loudongfeng)). +* Support detect output format by file exctension in `clickhouse-client` and `clickhouse-local`. [#61036](https://github.com/ClickHouse/ClickHouse/pull/61036) ([豪肥肥](https://github.com/HowePa)). +* Update memory limit in runtime when Linux's CGroups value changed. [#61049](https://github.com/ClickHouse/ClickHouse/pull/61049) ([Han Fei](https://github.com/hanfei1991)). +* Add the function `toUInt128OrZero`, which was missed by mistake (the mistake is related to https://github.com/ClickHouse/ClickHouse/pull/945). The compatibility aliases `FROM_UNIXTIME` and `DATE_FORMAT` (they are not ClickHouse-native and only exist for MySQL compatibility) have been made case insensitive, as expected for SQL-compatibility aliases. [#61114](https://github.com/ClickHouse/ClickHouse/pull/61114) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improvements for the access checks, allowing to revoke of unpossessed rights in case the target user doesn't have the revoking grants either. Example: `GRANT SELECT ON *.* TO user1; REVOKE SELECT ON system.* FROM user1;`. [#61115](https://github.com/ClickHouse/ClickHouse/pull/61115) ([pufit](https://github.com/pufit)). +* Fix `has()` function with `Nullable` column (fixes [#60214](https://github.com/ClickHouse/ClickHouse/issues/60214)). [#61249](https://github.com/ClickHouse/ClickHouse/pull/61249) ([Mikhail Koviazin](https://github.com/mkmkme)). +* Now it's possible to specify the attribute `merge="true"` in config substitutions for subtrees ``. In case this attribute specified, clickhouse will merge subtree with existing configuration, otherwise default behavior is append new content to configuration. [#61299](https://github.com/ClickHouse/ClickHouse/pull/61299) ([alesapin](https://github.com/alesapin)). +* Add async metrics for virtual memory mappings: `VMMaxMapCount` & `VMNumMaps`. Closes [#60662](https://github.com/ClickHouse/ClickHouse/issues/60662). [#61354](https://github.com/ClickHouse/ClickHouse/pull/61354) ([Tuan Pham Anh](https://github.com/tuanpavn)). +* Use `temporary_files_codec` setting in all places where we create temporary data, for example external memory sorting and external memory GROUP BY. Before it worked only in `partial_merge` JOIN algorithm. [#61456](https://github.com/ClickHouse/ClickHouse/pull/61456) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a new setting `max_parser_backtracks` which allows to limit the complexity of query parsing. [#61502](https://github.com/ClickHouse/ClickHouse/pull/61502) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Less contention during dynamic resize of the filesystem cache. [#61524](https://github.com/ClickHouse/ClickHouse/pull/61524) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disallow sharded mode of StorageS3 queue, because it will be rewritten. [#61537](https://github.com/ClickHouse/ClickHouse/pull/61537) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed typo: from `use_leagcy_max_level` to `use_legacy_max_level`. [#61545](https://github.com/ClickHouse/ClickHouse/pull/61545) ([William Schoeffel](https://github.com/wiledusc)). +* Remove some duplicate entries in `system.blob_storage_log`. [#61622](https://github.com/ClickHouse/ClickHouse/pull/61622) ([YenchangChan](https://github.com/YenchangChan)). +* Added `current_user` function as a compatibility alias for MySQL. [#61770](https://github.com/ClickHouse/ClickHouse/pull/61770) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Fix inconsistent floating point aggregate function states in mixed x86-64 / ARM clusters [#60610](https://github.com/ClickHouse/ClickHouse/pull/60610) ([Harry Lee](https://github.com/HarryLeeIBM)). + +#### Build/Testing/Packaging Improvement +* The real-time query profiler now works on AArch64. In previous versions, it worked only when a program didn't spend time inside a syscall. [#60807](https://github.com/ClickHouse/ClickHouse/pull/60807) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ClickHouse version has been added to docker labels. Closes [#54224](https://github.com/ClickHouse/ClickHouse/issues/54224). [#60949](https://github.com/ClickHouse/ClickHouse/pull/60949) ([Nikolay Monkov](https://github.com/nikmonkov)). +* Upgrade `prqlc` to 0.11.3. [#60616](https://github.com/ClickHouse/ClickHouse/pull/60616) ([Maximilian Roos](https://github.com/max-sixty)). +* Add generic query text fuzzer in `clickhouse-local`. [#61508](https://github.com/ClickHouse/ClickHouse/pull/61508) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Fix finished_mutations_to_keep=0 for MergeTree (as docs says 0 is to keep everything) [#60031](https://github.com/ClickHouse/ClickHouse/pull/60031) ([Azat Khuzhin](https://github.com/azat)). +* Something was wrong with the FINAL optimization, here is how the author describes it: "PartsSplitter invalid ranges for the same part". [#60041](https://github.com/ClickHouse/ClickHouse/pull/60041) ([Maksim Kita](https://github.com/kitaisreal)). +* Something was wrong with Apache Hive, which is experimental and not supported. [#60262](https://github.com/ClickHouse/ClickHouse/pull/60262) ([shanfengp](https://github.com/Aed-p)). +* An improvement for experimental parallel replicas: force reanalysis if parallel replicas changed [#60362](https://github.com/ClickHouse/ClickHouse/pull/60362) ([Raúl Marín](https://github.com/Algunenano)). +* Fix usage of plain metadata type with new disks configuration option [#60396](https://github.com/ClickHouse/ClickHouse/pull/60396) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Try to fix logical error 'Cannot capture column because it has incompatible type' in mapContainsKeyLike [#60451](https://github.com/ClickHouse/ClickHouse/pull/60451) ([Kruglov Pavel](https://github.com/Avogar)). +* Avoid calculation of scalar subqueries for CREATE TABLE. [#60464](https://github.com/ClickHouse/ClickHouse/pull/60464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix deadlock in parallel parsing when lots of rows are skipped due to errors [#60516](https://github.com/ClickHouse/ClickHouse/pull/60516) ([Kruglov Pavel](https://github.com/Avogar)). +* Something was wrong with experimental KQL (Kusto) support: fix `max_query_size_for_kql_compound_operator`: [#60534](https://github.com/ClickHouse/ClickHouse/pull/60534) ([Yong Wang](https://github.com/kashwy)). +* Keeper fix: add timeouts when waiting for commit logs [#60544](https://github.com/ClickHouse/ClickHouse/pull/60544) ([Antonio Andelic](https://github.com/antonio2368)). +* Don't output number tips for date types [#60577](https://github.com/ClickHouse/ClickHouse/pull/60577) ([Raúl Marín](https://github.com/Algunenano)). +* Fix reading from MergeTree with non-deterministic functions in filter [#60586](https://github.com/ClickHouse/ClickHouse/pull/60586) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix logical error on bad compatibility setting value type [#60596](https://github.com/ClickHouse/ClickHouse/pull/60596) ([Kruglov Pavel](https://github.com/Avogar)). +* fix(prql): Robust panic handler [#60615](https://github.com/ClickHouse/ClickHouse/pull/60615) ([Maximilian Roos](https://github.com/max-sixty)). +* Fix `intDiv` for decimal and date arguments [#60672](https://github.com/ClickHouse/ClickHouse/pull/60672) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Fix: expand CTE in alter modify query [#60682](https://github.com/ClickHouse/ClickHouse/pull/60682) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix system.parts for non-Atomic/Ordinary database engine (i.e. Memory) [#60689](https://github.com/ClickHouse/ClickHouse/pull/60689) ([Azat Khuzhin](https://github.com/azat)). +* Fix "Invalid storage definition in metadata file" for parameterized views [#60708](https://github.com/ClickHouse/ClickHouse/pull/60708) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer overflow in CompressionCodecMultiple [#60731](https://github.com/ClickHouse/ClickHouse/pull/60731) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove nonsense from SQL/JSON [#60738](https://github.com/ClickHouse/ClickHouse/pull/60738) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove wrong assertion in aggregate function quantileGK [#60740](https://github.com/ClickHouse/ClickHouse/pull/60740) ([李扬](https://github.com/taiyang-li)). +* Fix insert-select + insert_deduplication_token bug by setting streams to 1 [#60745](https://github.com/ClickHouse/ClickHouse/pull/60745) ([Jordi Villar](https://github.com/jrdi)). +* Prevent setting custom metadata headers on unsupported multipart upload operations [#60748](https://github.com/ClickHouse/ClickHouse/pull/60748) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)). +* Fix toStartOfInterval [#60763](https://github.com/ClickHouse/ClickHouse/pull/60763) ([Andrey Zvonov](https://github.com/zvonand)). +* Fix crash in arrayEnumerateRanked [#60764](https://github.com/ClickHouse/ClickHouse/pull/60764) ([Raúl Marín](https://github.com/Algunenano)). +* Fix crash when using input() in INSERT SELECT JOIN [#60765](https://github.com/ClickHouse/ClickHouse/pull/60765) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash with different allow_experimental_analyzer value in subqueries [#60770](https://github.com/ClickHouse/ClickHouse/pull/60770) ([Dmitry Novik](https://github.com/novikd)). +* Remove recursion when reading from S3 [#60849](https://github.com/ClickHouse/ClickHouse/pull/60849) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix possible stuck on error in HashedDictionaryParallelLoader [#60926](https://github.com/ClickHouse/ClickHouse/pull/60926) ([vdimir](https://github.com/vdimir)). +* Fix async RESTORE with Replicated database (experimental feature) [#60934](https://github.com/ClickHouse/ClickHouse/pull/60934) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix deadlock in async inserts to `Log` tables via native protocol [#61055](https://github.com/ClickHouse/ClickHouse/pull/61055) ([Anton Popov](https://github.com/CurtizJ)). +* Fix lazy execution of default argument in dictGetOrDefault for RangeHashedDictionary [#61196](https://github.com/ClickHouse/ClickHouse/pull/61196) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix multiple bugs in groupArraySorted [#61203](https://github.com/ClickHouse/ClickHouse/pull/61203) ([Raúl Marín](https://github.com/Algunenano)). +* Fix Keeper reconfig for standalone binary [#61233](https://github.com/ClickHouse/ClickHouse/pull/61233) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix usage of session_token in S3 engine [#61234](https://github.com/ClickHouse/ClickHouse/pull/61234) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix possible incorrect result of aggregate function `uniqExact` [#61257](https://github.com/ClickHouse/ClickHouse/pull/61257) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bugs in show database [#61269](https://github.com/ClickHouse/ClickHouse/pull/61269) ([Raúl Marín](https://github.com/Algunenano)). +* Fix logical error in RabbitMQ storage with MATERIALIZED columns [#61320](https://github.com/ClickHouse/ClickHouse/pull/61320) ([vdimir](https://github.com/vdimir)). +* Fix CREATE OR REPLACE DICTIONARY [#61356](https://github.com/ClickHouse/ClickHouse/pull/61356) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix ATTACH query with external ON CLUSTER [#61365](https://github.com/ClickHouse/ClickHouse/pull/61365) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix consecutive keys optimization for nullable keys [#61393](https://github.com/ClickHouse/ClickHouse/pull/61393) ([Anton Popov](https://github.com/CurtizJ)). +* fix issue of actions dag split [#61458](https://github.com/ClickHouse/ClickHouse/pull/61458) ([Raúl Marín](https://github.com/Algunenano)). +* Fix finishing a failed RESTORE [#61466](https://github.com/ClickHouse/ClickHouse/pull/61466) ([Vitaly Baranov](https://github.com/vitlibar)). +* Disable async_insert_use_adaptive_busy_timeout correctly with compatibility settings [#61468](https://github.com/ClickHouse/ClickHouse/pull/61468) ([Raúl Marín](https://github.com/Algunenano)). +* Allow queuing in restore pool [#61475](https://github.com/ClickHouse/ClickHouse/pull/61475) ([Nikita Taranov](https://github.com/nickitat)). +* Fix an inconsistency when reading system.parts using UUID. [#61479](https://github.com/ClickHouse/ClickHouse/pull/61479) ([Dan Wu](https://github.com/wudanzy)). +* Fix ALTER QUERY MODIFY SQL SECURITY [#61480](https://github.com/ClickHouse/ClickHouse/pull/61480) ([pufit](https://github.com/pufit)). +* Fix a crash in window view (experimental feature) [#61526](https://github.com/ClickHouse/ClickHouse/pull/61526) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `repeat` with non-native integers [#61527](https://github.com/ClickHouse/ClickHouse/pull/61527) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix client's `-s` argument [#61530](https://github.com/ClickHouse/ClickHouse/pull/61530) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix crash in arrayPartialReverseSort [#61539](https://github.com/ClickHouse/ClickHouse/pull/61539) ([Raúl Marín](https://github.com/Algunenano)). +* Fix string search with const position [#61547](https://github.com/ClickHouse/ClickHouse/pull/61547) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix addDays cause an error when used DateTime64 [#61561](https://github.com/ClickHouse/ClickHouse/pull/61561) ([Shuai li](https://github.com/loneylee)). +* Disallow LowCardinality input type for JSONExtract [#61617](https://github.com/ClickHouse/ClickHouse/pull/61617) ([Julia Kartseva](https://github.com/jkartseva)). +* Fix `system.part_log` for async insert with deduplication [#61620](https://github.com/ClickHouse/ClickHouse/pull/61620) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix a `Non-ready set` exception for system.parts. [#61666](https://github.com/ClickHouse/ClickHouse/pull/61666) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix actual_part_name for REPLACE_RANGE (`Entry actual part isn't empty yet`) [#61675](https://github.com/ClickHouse/ClickHouse/pull/61675) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix a sanitizer report in `multiSearchAllPositionsCaseInsensitiveUTF8` for incorrect UTF-8 [#61749](https://github.com/ClickHouse/ClickHouse/pull/61749) ([pufit](https://github.com/pufit)). +* Fix an observation that the RANGE frame is not supported for Nullable columns. [#61766](https://github.com/ClickHouse/ClickHouse/pull/61766) ([YuanLiu](https://github.com/ditgittube)). + ### ClickHouse release 24.2, 2024-02-29 #### Backward Incompatible Change diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c4e16eace2..42c21cae9f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,11 +61,16 @@ if (ENABLE_CHECK_HEAVY_BUILDS) # set CPU time limit to 1000 seconds set (RLIMIT_CPU 1000) - # -fsanitize=memory and address are too heavy + # Sanitizers are too heavy if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE) set (RLIMIT_DATA 10000000000) # 10G endif() + # For some files currently building RISCV64 might be too slow. TODO: Improve compilation times per file + if (ARCH_RISCV64) + set (RLIMIT_CPU 1800) + endif() + set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) endif () @@ -102,6 +107,8 @@ if (ENABLE_FUZZING) # For codegen_select_fuzzer set (ENABLE_PROTOBUF 1) + + add_compile_definitions(FUZZING_MODE=1) endif() # Global libraries @@ -574,7 +581,7 @@ if (FUZZER) if (NOT(target_type STREQUAL "INTERFACE_LIBRARY" OR target_type STREQUAL "UTILITY")) target_compile_options(${target} PRIVATE "-fsanitize=fuzzer-no-link") endif() - if (target_type STREQUAL "EXECUTABLE" AND (target MATCHES ".+_fuzzer" OR target STREQUAL "clickhouse")) + if (target_type STREQUAL "EXECUTABLE" AND target MATCHES ".+_fuzzer") message(STATUS "${target} instrumented with fuzzer") target_link_libraries(${target} PUBLIC ch_contrib::fuzzer) # Add to fuzzers bundle @@ -583,6 +590,12 @@ if (FUZZER) get_target_property(target_bin_dir ${target} BINARY_DIR) add_custom_command(TARGET fuzzers POST_BUILD COMMAND mv "${target_bin_dir}/${target_bin_name}" "${CMAKE_CURRENT_BINARY_DIR}/programs/" VERBATIM) endif() + if (target STREQUAL "clickhouse") + message(STATUS "${target} instrumented with fuzzer") + target_link_libraries(${target} PUBLIC ch_contrib::fuzzer_no_main) + # Add to fuzzers bundle + add_dependencies(fuzzers ${target}) + endif() endif() endforeach() add_custom_command(TARGET fuzzers POST_BUILD COMMAND SRC=${CMAKE_SOURCE_DIR} BIN=${CMAKE_BINARY_DIR} OUT=${CMAKE_BINARY_DIR}/programs ${CMAKE_SOURCE_DIR}/tests/fuzz/build.sh VERBATIM) diff --git a/README.md b/README.md index e00ce42a60b..2b97bd25d70 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,6 @@ curl https://clickhouse.com/ | sh * [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time. * [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events. * [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev. -* [Static Analysis (SonarCloud)](https://sonarcloud.io/project/issues?resolved=false&id=ClickHouse_ClickHouse) proposes C++ quality improvements. * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any. ## Monthly Release & Community Call diff --git a/SECURITY.md b/SECURITY.md index 86578b188d8..4701f2ec70b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -13,18 +13,16 @@ The following versions of ClickHouse server are currently being supported with s | Version | Supported | |:-|:-| +| 24.3 | ✔️ | | 24.2 | ✔️ | | 24.1 | ✔️ | -| 23.12 | ✔️ | -| 23.11 | ❌ | -| 23.10 | ❌ | -| 23.9 | ❌ | +| 23.* | ❌ | | 23.8 | ✔️ | | 23.7 | ❌ | | 23.6 | ❌ | | 23.5 | ❌ | | 23.4 | ❌ | -| 23.3 | ✔️ | +| 23.3 | ❌ | | 23.2 | ❌ | | 23.1 | ❌ | | 22.* | ❌ | diff --git a/base/base/IPv4andIPv6.h b/base/base/IPv4andIPv6.h index e2f93b54124..9b1e518c161 100644 --- a/base/base/IPv4andIPv6.h +++ b/base/base/IPv4andIPv6.h @@ -1,8 +1,7 @@ #pragma once -#include #include -#include +#include #include namespace DB @@ -62,7 +61,8 @@ namespace std { size_t operator()(const DB::IPv6 & x) const { - return std::hash{}(std::string_view(reinterpret_cast(&x.toUnderType()), IPV6_BINARY_LENGTH)); + return std::hash{}( + std::string_view(reinterpret_cast(&x.toUnderType()), sizeof(DB::IPv6::UnderlyingType))); } }; diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 17b1fa7cd6a..0e98b6e5ee6 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -13,8 +13,6 @@ #include #include -#include - // NOLINTBEGIN(*) /// Use same extended double for all platforms @@ -22,6 +20,7 @@ #define CONSTEXPR_FROM_DOUBLE constexpr using FromDoubleIntermediateType = long double; #else +#include #include /// `wide_integer_from_builtin` can't be constexpr with non-literal `cpp_bin_float_double_extended` #define CONSTEXPR_FROM_DOUBLE @@ -309,6 +308,13 @@ struct integer::_impl constexpr uint64_t max_int = std::numeric_limits::max(); static_assert(std::is_same_v || std::is_same_v); /// Implementation specific behaviour on overflow (if we don't check here, stack overflow will triggered in bigint_cast). +#if (LDBL_MANT_DIG == 64) + if (!std::isfinite(t)) + { + self = 0; + return; + } +#else if constexpr (std::is_same_v) { if (!std::isfinite(t)) @@ -325,6 +331,7 @@ struct integer::_impl return; } } +#endif const T alpha = t / static_cast(max_int); diff --git a/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h b/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h new file mode 100644 index 00000000000..7b31307e1ca --- /dev/null +++ b/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h @@ -0,0 +1,75 @@ +// +// FPEnvironment_SUN.h +// +// Library: Foundation +// Package: Core +// Module: FPEnvironment +// +// Definitions of class FPEnvironmentImpl for Solaris. +// +// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Foundation_FPEnvironment_SUN_INCLUDED +#define Foundation_FPEnvironment_SUN_INCLUDED + + +#include +#include "Poco/Foundation.h" + + +namespace Poco +{ + + +class FPEnvironmentImpl +{ +protected: + enum RoundingModeImpl + { + FP_ROUND_DOWNWARD_IMPL = FP_RM, + FP_ROUND_UPWARD_IMPL = FP_RP, + FP_ROUND_TONEAREST_IMPL = FP_RN, + FP_ROUND_TOWARDZERO_IMPL = FP_RZ + }; + enum FlagImpl + { + FP_DIVIDE_BY_ZERO_IMPL = FP_X_DZ, + FP_INEXACT_IMPL = FP_X_IMP, + FP_OVERFLOW_IMPL = FP_X_OFL, + FP_UNDERFLOW_IMPL = FP_X_UFL, + FP_INVALID_IMPL = FP_X_INV + }; + FPEnvironmentImpl(); + FPEnvironmentImpl(const FPEnvironmentImpl & env); + ~FPEnvironmentImpl(); + FPEnvironmentImpl & operator=(const FPEnvironmentImpl & env); + void keepCurrentImpl(); + static void clearFlagsImpl(); + static bool isFlagImpl(FlagImpl flag); + static void setRoundingModeImpl(RoundingModeImpl mode); + static RoundingModeImpl getRoundingModeImpl(); + static bool isInfiniteImpl(float value); + static bool isInfiniteImpl(double value); + static bool isInfiniteImpl(long double value); + static bool isNaNImpl(float value); + static bool isNaNImpl(double value); + static bool isNaNImpl(long double value); + static float copySignImpl(float target, float source); + static double copySignImpl(double target, double source); + static long double copySignImpl(long double target, long double source); + +private: + fp_rnd _rnd; + fp_except _exc; +}; + + +} // namespace Poco + + +#endif // Foundation_FPEnvironment_SUN_INCLUDED diff --git a/base/poco/Foundation/src/Environment_UNIX.cpp b/base/poco/Foundation/src/Environment_UNIX.cpp index 202e5d88f83..faabb374778 100644 --- a/base/poco/Foundation/src/Environment_UNIX.cpp +++ b/base/poco/Foundation/src/Environment_UNIX.cpp @@ -281,15 +281,15 @@ void EnvironmentImpl::nodeIdImpl(NodeId& id) /// #include #if defined(sun) || defined(__sun) #include +#include +#include +#include #endif /// #include /// #include /// #include /// #include /// #include -/// #include -/// #include -/// #include /// #include diff --git a/base/poco/Foundation/src/FPEnvironment_SUN.cpp b/base/poco/Foundation/src/FPEnvironment_SUN.cpp new file mode 100644 index 00000000000..36ee36431df --- /dev/null +++ b/base/poco/Foundation/src/FPEnvironment_SUN.cpp @@ -0,0 +1,139 @@ +// +// FPEnvironment_SUN.cpp +// +// Library: Foundation +// Package: Core +// Module: FPEnvironment +// +// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include +#include "Poco/FPEnvironment_SUN.h" + + +namespace Poco { + + +FPEnvironmentImpl::FPEnvironmentImpl() +{ + _rnd = fpgetround(); + _exc = fpgetmask(); +} + + +FPEnvironmentImpl::FPEnvironmentImpl(const FPEnvironmentImpl& env) +{ + _rnd = env._rnd; + _exc = env._exc; +} + + +FPEnvironmentImpl::~FPEnvironmentImpl() +{ + fpsetround(_rnd); + fpsetmask(_exc); +} + + +FPEnvironmentImpl& FPEnvironmentImpl::operator = (const FPEnvironmentImpl& env) +{ + _rnd = env._rnd; + _exc = env._exc; + return *this; +} + + +bool FPEnvironmentImpl::isInfiniteImpl(float value) +{ + int cls = fpclass(value); + return cls == FP_PINF || cls == FP_NINF; +} + + +bool FPEnvironmentImpl::isInfiniteImpl(double value) +{ + int cls = fpclass(value); + return cls == FP_PINF || cls == FP_NINF; +} + + +bool FPEnvironmentImpl::isInfiniteImpl(long double value) +{ + int cls = fpclass(value); + return cls == FP_PINF || cls == FP_NINF; +} + + +bool FPEnvironmentImpl::isNaNImpl(float value) +{ + return isnanf(value) != 0; +} + + +bool FPEnvironmentImpl::isNaNImpl(double value) +{ + return isnan(value) != 0; +} + + +bool FPEnvironmentImpl::isNaNImpl(long double value) +{ + return isnan((double) value) != 0; +} + + +float FPEnvironmentImpl::copySignImpl(float target, float source) +{ + return (float) copysign(target, source); +} + + +double FPEnvironmentImpl::copySignImpl(double target, double source) +{ + return (float) copysign(target, source); +} + + +long double FPEnvironmentImpl::copySignImpl(long double target, long double source) +{ + return (source > 0 && target > 0) || (source < 0 && target < 0) ? target : -target; +} + + +void FPEnvironmentImpl::keepCurrentImpl() +{ + fpsetround(_rnd); + fpsetmask(_exc); +} + + +void FPEnvironmentImpl::clearFlagsImpl() +{ + fpsetsticky(0); +} + + +bool FPEnvironmentImpl::isFlagImpl(FlagImpl flag) +{ + return (fpgetsticky() & flag) != 0; +} + + +void FPEnvironmentImpl::setRoundingModeImpl(RoundingModeImpl mode) +{ + fpsetround((fp_rnd) mode); +} + + +FPEnvironmentImpl::RoundingModeImpl FPEnvironmentImpl::getRoundingModeImpl() +{ + return (FPEnvironmentImpl::RoundingModeImpl) fpgetround(); +} + + +} // namespace Poco diff --git a/base/poco/Foundation/src/NamedEvent_UNIX.cpp b/base/poco/Foundation/src/NamedEvent_UNIX.cpp index 978e6e0bc02..3cda4104c73 100644 --- a/base/poco/Foundation/src/NamedEvent_UNIX.cpp +++ b/base/poco/Foundation/src/NamedEvent_UNIX.cpp @@ -31,7 +31,7 @@ namespace Poco { -#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) +#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) || (POCO_OS == POCO_OS_SOLARIS) union semun { int val; diff --git a/base/poco/Foundation/src/NamedMutex_UNIX.cpp b/base/poco/Foundation/src/NamedMutex_UNIX.cpp index 6cfa1369c9d..d53d54d7bb5 100644 --- a/base/poco/Foundation/src/NamedMutex_UNIX.cpp +++ b/base/poco/Foundation/src/NamedMutex_UNIX.cpp @@ -31,7 +31,7 @@ namespace Poco { -#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) +#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) || (POCO_OS == POCO_OS_SOLARIS) union semun { int val; diff --git a/base/poco/Foundation/src/pcre_compile.c b/base/poco/Foundation/src/pcre_compile.c index 3a6fafe8d56..b5f5f9a8286 100644 --- a/base/poco/Foundation/src/pcre_compile.c +++ b/base/poco/Foundation/src/pcre_compile.c @@ -4835,7 +4835,7 @@ for (;; ptr++) If the class contains characters outside the 0-255 range, a different opcode is compiled. It may optionally have a bit map for characters < 256, - but those above are are explicitly listed afterwards. A flag byte tells + but those above are explicitly listed afterwards. A flag byte tells whether the bitmap is present, and whether this is a negated class or not. In JavaScript compatibility mode, an isolated ']' causes an error. In diff --git a/base/poco/JSON/src/pdjson.c b/base/poco/JSON/src/pdjson.c index 18768ac96d3..563fa277439 100644 --- a/base/poco/JSON/src/pdjson.c +++ b/base/poco/JSON/src/pdjson.c @@ -314,13 +314,13 @@ static int read_unicode(json_stream *json) if (l < 0xdc00 || l > 0xdfff) { json_error(json, "invalid surrogate pair continuation \\u%04lx out " - "of range (dc00-dfff)", l); + "of range (dc00-dfff)", (unsigned long)l); return -1; } cp = ((h - 0xd800) * 0x400) + ((l - 0xdc00) + 0x10000); } else if (cp >= 0xdc00 && cp <= 0xdfff) { - json_error(json, "dangling surrogate \\u%04lx", cp); + json_error(json, "dangling surrogate \\u%04lx", (unsigned long)cp); return -1; } diff --git a/base/poco/Net/CMakeLists.txt b/base/poco/Net/CMakeLists.txt index 792045c9b43..50ffbdf905a 100644 --- a/base/poco/Net/CMakeLists.txt +++ b/base/poco/Net/CMakeLists.txt @@ -9,6 +9,10 @@ elseif (OS_DARWIN OR OS_FREEBSD) target_compile_definitions (_poco_net PUBLIC POCO_HAVE_FD_POLL) endif () +if (OS_SUNOS) + target_link_libraries (_poco_net PUBLIC socket nsl) +endif () + # TODO: remove these warning exclusions target_compile_options (_poco_net PRIVATE diff --git a/base/poco/Net/include/Poco/Net/HTTPClientSession.h b/base/poco/Net/include/Poco/Net/HTTPClientSession.h index 1cef988566c..edbb135d8c6 100644 --- a/base/poco/Net/include/Poco/Net/HTTPClientSession.h +++ b/base/poco/Net/include/Poco/Net/HTTPClientSession.h @@ -213,6 +213,19 @@ namespace Net Poco::Timespan getKeepAliveTimeout() const; /// Returns the connection timeout for HTTP connections. + void setKeepAliveMaxRequests(int max_requests); + + int getKeepAliveMaxRequests() const; + + int getKeepAliveRequest() const; + + bool isKeepAliveExpired(double reliability = 1.0) const; + /// Returns if the connection is expired with some margin as fraction of timeout as reliability + + double getKeepAliveReliability() const; + /// Returns the current fraction of keep alive timeout when connection is considered safe to use + /// It helps to avoid situation when a client uses nearly expired connection and receives NoMessageException + virtual std::ostream & sendRequest(HTTPRequest & request); /// Sends the header for the given HTTP request to /// the server. @@ -345,6 +358,8 @@ namespace Net void assign(HTTPClientSession & session); + void setKeepAliveRequest(int request); + HTTPSessionFactory _proxySessionFactory; /// Factory to create HTTPClientSession to proxy. private: @@ -353,6 +368,8 @@ namespace Net Poco::UInt16 _port; ProxyConfig _proxyConfig; Poco::Timespan _keepAliveTimeout; + int _keepAliveCurrentRequest = 0; + int _keepAliveMaxRequests = 1000; Poco::Timestamp _lastRequest; bool _reconnect; bool _mustReconnect; @@ -361,6 +378,7 @@ namespace Net Poco::SharedPtr _pRequestStream; Poco::SharedPtr _pResponseStream; + static const double _defaultKeepAliveReliabilityLevel; static ProxyConfig _globalProxyConfig; HTTPClientSession(const HTTPClientSession &); @@ -450,9 +468,19 @@ namespace Net return _lastRequest; } - inline void HTTPClientSession::setLastRequest(Poco::Timestamp time) + inline double HTTPClientSession::getKeepAliveReliability() const { - _lastRequest = time; + return _defaultKeepAliveReliabilityLevel; + } + + inline int HTTPClientSession::getKeepAliveMaxRequests() const + { + return _keepAliveMaxRequests; + } + + inline int HTTPClientSession::getKeepAliveRequest() const + { + return _keepAliveCurrentRequest; } } diff --git a/base/poco/Net/include/Poco/Net/HTTPMessage.h b/base/poco/Net/include/Poco/Net/HTTPMessage.h index 0bef50803a8..8bc95ccc1af 100644 --- a/base/poco/Net/include/Poco/Net/HTTPMessage.h +++ b/base/poco/Net/include/Poco/Net/HTTPMessage.h @@ -120,6 +120,10 @@ namespace Net /// The value is set to "Keep-Alive" if keepAlive is /// true, or to "Close" otherwise. + void setKeepAliveTimeout(int timeout, int max_requests); + int getKeepAliveTimeout() const; + int getKeepAliveMaxRequests() const; + bool getKeepAlive() const; /// Returns true if /// * the message has a Connection header field and its value is "Keep-Alive" diff --git a/base/poco/Net/include/Poco/Net/HTTPServerParams.h b/base/poco/Net/include/Poco/Net/HTTPServerParams.h index 3c836a630a0..d614c62d57a 100644 --- a/base/poco/Net/include/Poco/Net/HTTPServerParams.h +++ b/base/poco/Net/include/Poco/Net/HTTPServerParams.h @@ -44,7 +44,7 @@ namespace Net /// - timeout: 60 seconds /// - keepAlive: true /// - maxKeepAliveRequests: 0 - /// - keepAliveTimeout: 10 seconds + /// - keepAliveTimeout: 15 seconds void setServerName(const std::string & serverName); /// Sets the name and port (name:port) that the server uses to identify itself. diff --git a/base/poco/Net/include/Poco/Net/HTTPServerSession.h b/base/poco/Net/include/Poco/Net/HTTPServerSession.h index ec928af304f..3df7995509a 100644 --- a/base/poco/Net/include/Poco/Net/HTTPServerSession.h +++ b/base/poco/Net/include/Poco/Net/HTTPServerSession.h @@ -56,6 +56,8 @@ namespace Net SocketAddress serverAddress(); /// Returns the server's address. + void setKeepAliveTimeout(Poco::Timespan keepAliveTimeout); + private: bool _firstRequest; Poco::Timespan _keepAliveTimeout; diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp index 33a3dcc4901..c9899266be7 100644 --- a/base/poco/Net/src/HTTPClientSession.cpp +++ b/base/poco/Net/src/HTTPClientSession.cpp @@ -37,6 +37,7 @@ namespace Net { HTTPClientSession::ProxyConfig HTTPClientSession::_globalProxyConfig; +const double HTTPClientSession::_defaultKeepAliveReliabilityLevel = 0.9; HTTPClientSession::HTTPClientSession(): @@ -220,7 +221,41 @@ void HTTPClientSession::setGlobalProxyConfig(const ProxyConfig& config) void HTTPClientSession::setKeepAliveTimeout(const Poco::Timespan& timeout) { - _keepAliveTimeout = timeout; + if (connected()) + { + throw Poco::IllegalStateException("cannot change keep alive timeout on initiated connection, " + "That value is managed privately after connection is established."); + } + _keepAliveTimeout = timeout; +} + + +void HTTPClientSession::setKeepAliveMaxRequests(int max_requests) +{ + if (connected()) + { + throw Poco::IllegalStateException("cannot change keep alive max requests on initiated connection, " + "That value is managed privately after connection is established."); + } + _keepAliveMaxRequests = max_requests; +} + + +void HTTPClientSession::setKeepAliveRequest(int request) +{ + _keepAliveCurrentRequest = request; +} + + + +void HTTPClientSession::setLastRequest(Poco::Timestamp time) +{ + if (connected()) + { + throw Poco::IllegalStateException("cannot change last request on initiated connection, " + "That value is managed privately after connection is established."); + } + _lastRequest = time; } @@ -231,6 +266,8 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request) clearException(); _responseReceived = false; + _keepAliveCurrentRequest += 1; + bool keepAlive = getKeepAlive(); if (((connected() && !keepAlive) || mustReconnect()) && !_host.empty()) { @@ -241,8 +278,10 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request) { if (!connected()) reconnect(); - if (!keepAlive) - request.setKeepAlive(false); + if (!request.has(HTTPMessage::CONNECTION)) + request.setKeepAlive(keepAlive); + if (keepAlive && !request.has(HTTPMessage::CONNECTION_KEEP_ALIVE) && _keepAliveTimeout.totalSeconds() > 0) + request.setKeepAliveTimeout(_keepAliveTimeout.totalSeconds(), _keepAliveMaxRequests); if (!request.has(HTTPRequest::HOST) && !_host.empty()) request.setHost(_host, _port); if (!_proxyConfig.host.empty() && !bypassProxy()) @@ -324,6 +363,17 @@ std::istream& HTTPClientSession::receiveResponse(HTTPResponse& response) _mustReconnect = getKeepAlive() && !response.getKeepAlive(); + if (!_mustReconnect) + { + /// when server sends its keep alive timeout, client has to follow that value + auto timeout = response.getKeepAliveTimeout(); + if (timeout > 0) + _keepAliveTimeout = std::min(_keepAliveTimeout, Poco::Timespan(timeout, 0)); + auto max_requests = response.getKeepAliveMaxRequests(); + if (max_requests > 0) + _keepAliveMaxRequests = std::min(_keepAliveMaxRequests, max_requests); + } + if (!_expectResponseBody || response.getStatus() < 200 || response.getStatus() == HTTPResponse::HTTP_NO_CONTENT || response.getStatus() == HTTPResponse::HTTP_NOT_MODIFIED) _pResponseStream = new HTTPFixedLengthInputStream(*this, 0); else if (response.getChunkedTransferEncoding()) @@ -430,15 +480,18 @@ std::string HTTPClientSession::proxyRequestPrefix() const return result; } +bool HTTPClientSession::isKeepAliveExpired(double reliability) const +{ + Poco::Timestamp now; + return Timespan(Timestamp::TimeDiff(reliability *_keepAliveTimeout.totalMicroseconds())) <= now - _lastRequest + || _keepAliveCurrentRequest > _keepAliveMaxRequests; +} bool HTTPClientSession::mustReconnect() const { if (!_mustReconnect) - { - Poco::Timestamp now; - return _keepAliveTimeout <= now - _lastRequest; - } - else return true; + return isKeepAliveExpired(_defaultKeepAliveReliabilityLevel); + return true; } @@ -511,14 +564,21 @@ void HTTPClientSession::assign(Poco::Net::HTTPClientSession & session) if (buffered()) throw Poco::LogicException("assign to a session with not empty buffered data"); - attachSocket(session.detachSocket()); - setLastRequest(session.getLastRequest()); + poco_assert(!connected()); + setResolvedHost(session.getResolvedHost()); - setKeepAlive(session.getKeepAlive()); + setProxyConfig(session.getProxyConfig()); setTimeout(session.getConnectionTimeout(), session.getSendTimeout(), session.getReceiveTimeout()); + setKeepAlive(session.getKeepAlive()); + + setLastRequest(session.getLastRequest()); setKeepAliveTimeout(session.getKeepAliveTimeout()); - setProxyConfig(session.getProxyConfig()); + + _keepAliveMaxRequests = session._keepAliveMaxRequests; + _keepAliveCurrentRequest = session._keepAliveCurrentRequest; + + attachSocket(session.detachSocket()); session.reset(); } diff --git a/base/poco/Net/src/HTTPMessage.cpp b/base/poco/Net/src/HTTPMessage.cpp index 0cd234ee9cb..c0083ec410c 100644 --- a/base/poco/Net/src/HTTPMessage.cpp +++ b/base/poco/Net/src/HTTPMessage.cpp @@ -17,6 +17,7 @@ #include "Poco/NumberFormatter.h" #include "Poco/NumberParser.h" #include "Poco/String.h" +#include using Poco::NumberFormatter; @@ -179,4 +180,51 @@ bool HTTPMessage::getKeepAlive() const } +void HTTPMessage::setKeepAliveTimeout(int timeout, int max_requests) +{ + add(HTTPMessage::CONNECTION_KEEP_ALIVE, std::format("timeout={}, max={}", timeout, max_requests)); +} + + +int parseFromHeaderValues(const std::string_view header_value, const std::string_view param_name) +{ + auto param_value_pos = header_value.find(param_name); + if (param_value_pos == std::string::npos) + param_value_pos = header_value.size(); + if (param_value_pos != header_value.size()) + param_value_pos += param_name.size(); + + auto param_value_end = header_value.find(',', param_value_pos); + if (param_value_end == std::string::npos) + param_value_end = header_value.size(); + + auto timeout_value_substr = header_value.substr(param_value_pos, param_value_end - param_value_pos); + if (timeout_value_substr.empty()) + return -1; + + int value = 0; + auto [ptr, ec] = std::from_chars(timeout_value_substr.begin(), timeout_value_substr.end(), value); + + if (ec == std::errc()) + return value; + + return -1; +} + + +int HTTPMessage::getKeepAliveTimeout() const +{ + const std::string& ka_header = get(HTTPMessage::CONNECTION_KEEP_ALIVE, HTTPMessage::EMPTY); + static const std::string_view timeout_param = "timeout="; + return parseFromHeaderValues(ka_header, timeout_param); +} + + +int HTTPMessage::getKeepAliveMaxRequests() const +{ + const std::string& ka_header = get(HTTPMessage::CONNECTION_KEEP_ALIVE, HTTPMessage::EMPTY); + static const std::string_view timeout_param = "max="; + return parseFromHeaderValues(ka_header, timeout_param); +} + } } // namespace Poco::Net diff --git a/base/poco/Net/src/HTTPServerConnection.cpp b/base/poco/Net/src/HTTPServerConnection.cpp index c57984b0162..d5eb29d3134 100644 --- a/base/poco/Net/src/HTTPServerConnection.cpp +++ b/base/poco/Net/src/HTTPServerConnection.cpp @@ -88,7 +88,18 @@ void HTTPServerConnection::run() pHandler->handleRequest(request, response); session.setKeepAlive(_pParams->getKeepAlive() && response.getKeepAlive() && session.canKeepAlive()); - } + + /// all that fuzz is all about to make session close with less timeout than 15s (set in HTTPServerParams c-tor) + if (_pParams->getKeepAlive() && response.getKeepAlive() && session.canKeepAlive()) + { + int value = response.getKeepAliveTimeout(); + if (value < 0) + value = request.getKeepAliveTimeout(); + if (value > 0) + session.setKeepAliveTimeout(Poco::Timespan(value, 0)); + } + + } else sendErrorResponse(session, HTTPResponse::HTTP_NOT_IMPLEMENTED); } catch (Poco::Exception&) diff --git a/base/poco/Net/src/HTTPServerSession.cpp b/base/poco/Net/src/HTTPServerSession.cpp index d4f2b24879e..f67a63a9e0e 100644 --- a/base/poco/Net/src/HTTPServerSession.cpp +++ b/base/poco/Net/src/HTTPServerSession.cpp @@ -33,6 +33,12 @@ HTTPServerSession::~HTTPServerSession() { } +void HTTPServerSession::setKeepAliveTimeout(Poco::Timespan keepAliveTimeout) +{ + _keepAliveTimeout = keepAliveTimeout; +} + + bool HTTPServerSession::hasMoreRequests() { diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 2929c64ded8..26cb0eb23c6 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54484) +SET(VERSION_REVISION 54485) SET(VERSION_MAJOR 24) -SET(VERSION_MINOR 3) +SET(VERSION_MINOR 4) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH 891689a41506d00aa169548f5b4a8774351242c4) -SET(VERSION_DESCRIBE v24.3.1.1-testing) -SET(VERSION_STRING 24.3.1.1) +SET(VERSION_GITHASH 2c5c589a882ceec35439650337b92db3e76f0081) +SET(VERSION_DESCRIBE v24.4.1.1-testing) +SET(VERSION_STRING 24.4.1.1) # end of autochange diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 88dea294bf5..9d53b2004b4 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -30,7 +30,7 @@ if (SANITIZE) elseif (SANITIZE STREQUAL "thread") set (TSAN_FLAGS "-fsanitize=thread") if (COMPILER_CLANG) - set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${PROJECT_SOURCE_DIR}/tests/tsan_suppressions.txt") + set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/tsan_ignorelist.txt") endif() set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}") @@ -48,7 +48,7 @@ if (SANITIZE) set(UBSAN_FLAGS "${UBSAN_FLAGS} -fno-sanitize=unsigned-integer-overflow") endif() if (COMPILER_CLANG) - set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${PROJECT_SOURCE_DIR}/tests/ubsan_suppressions.txt") + set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/ubsan_ignorelist.txt") endif() set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}") diff --git a/contrib/NuRaft b/contrib/NuRaft index 4a12f99dfc9..cb5dc3c906e 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 4a12f99dfc9d47c687ff7700b927cc76856225d1 +Subproject commit cb5dc3c906e80f253e9ce9535807caef827cc2e0 diff --git a/contrib/arrow b/contrib/arrow index ba5c67934e8..8f36d71d185 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit ba5c67934e8274d649befcffab56731632dc5253 +Subproject commit 8f36d71d18587f1f315ec832f424183cb6519cbb diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt index 63b3854eef9..96f740b6dd2 100644 --- a/contrib/avro-cmake/CMakeLists.txt +++ b/contrib/avro-cmake/CMakeLists.txt @@ -59,12 +59,3 @@ target_link_libraries (_avrocpp PRIVATE boost::headers_only boost::iostreams) target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR}) target_link_libraries (_avrocpp PRIVATE ch_contrib::snappy) - -# create a symlink to include headers with -set(AVRO_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include") -ADD_CUSTOM_TARGET(avro_symlink_headers ALL - COMMAND ${CMAKE_COMMAND} -E make_directory "${AVRO_INCLUDE_DIR}" - COMMAND ${CMAKE_COMMAND} -E create_symlink "${AVROCPP_ROOT_DIR}/api" "${AVRO_INCLUDE_DIR}/avro" -) -add_dependencies(_avrocpp avro_symlink_headers) -target_include_directories(_avrocpp SYSTEM BEFORE PUBLIC "${AVRO_INCLUDE_DIR}") diff --git a/contrib/c-ares-cmake/CMakeLists.txt b/contrib/c-ares-cmake/CMakeLists.txt index 86ab6f90260..daec96ff1b1 100644 --- a/contrib/c-ares-cmake/CMakeLists.txt +++ b/contrib/c-ares-cmake/CMakeLists.txt @@ -86,6 +86,8 @@ elseif (OS_DARWIN) target_compile_definitions(_c-ares PRIVATE -D_DARWIN_C_SOURCE) elseif (OS_FREEBSD) target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/freebsd") +elseif (OS_SUNOS) + target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/solaris") endif() add_library(ch_contrib::c-ares ALIAS _c-ares) diff --git a/contrib/c-ares-cmake/solaris/ares_build.h b/contrib/c-ares-cmake/solaris/ares_build.h new file mode 100644 index 00000000000..f42b59d07bd --- /dev/null +++ b/contrib/c-ares-cmake/solaris/ares_build.h @@ -0,0 +1,104 @@ +/* include/ares_build.h. Generated from ares_build.h.in by configure. */ +#ifndef __CARES_BUILD_H +#define __CARES_BUILD_H + + +/* Copyright (C) 2009 - 2021 by Daniel Stenberg et al + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose and without fee is hereby granted, provided + * that the above copyright notice appear in all copies and that both that + * copyright notice and this permission notice appear in supporting + * documentation, and that the name of M.I.T. not be used in advertising or + * publicity pertaining to distribution of the software without specific, + * written prior permission. M.I.T. makes no representations about the + * suitability of this software for any purpose. It is provided "as is" + * without express or implied warranty. + */ + +/* ================================================================ */ +/* NOTES FOR CONFIGURE CAPABLE SYSTEMS */ +/* ================================================================ */ + +/* + * NOTE 1: + * ------- + * + * Nothing in this file is intended to be modified or adjusted by the + * c-ares library user nor by the c-ares library builder. + * + * If you think that something actually needs to be changed, adjusted + * or fixed in this file, then, report it on the c-ares development + * mailing list: http://lists.haxx.se/listinfo/c-ares/ + * + * This header file shall only export symbols which are 'cares' or 'CARES' + * prefixed, otherwise public name space would be polluted. + * + * NOTE 2: + * ------- + * + * Right now you might be staring at file ares_build.h.in or ares_build.h, + * this is due to the following reason: + * + * On systems capable of running the configure script, the configure process + * will overwrite the distributed ares_build.h file with one that is suitable + * and specific to the library being configured and built, which is generated + * from the ares_build.h.in template file. + * + */ + +/* ================================================================ */ +/* DEFINITION OF THESE SYMBOLS SHALL NOT TAKE PLACE ANYWHERE ELSE */ +/* ================================================================ */ + +#ifdef CARES_TYPEOF_ARES_SOCKLEN_T +# error "CARES_TYPEOF_ARES_SOCKLEN_T shall not be defined except in ares_build.h" + Error Compilation_aborted_CARES_TYPEOF_ARES_SOCKLEN_T_already_defined +#endif + +#define CARES_HAVE_ARPA_NAMESER_H 1 +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H 1 + +/* ================================================================ */ +/* EXTERNAL INTERFACE SETTINGS FOR CONFIGURE CAPABLE SYSTEMS ONLY */ +/* ================================================================ */ + +/* Configure process defines this to 1 when it finds out that system */ +/* header file ws2tcpip.h must be included by the external interface. */ +/* #undef CARES_PULL_WS2TCPIP_H */ +#ifdef CARES_PULL_WS2TCPIP_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# include +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/types.h must be included by the external interface. */ +#define CARES_PULL_SYS_TYPES_H 1 +#ifdef CARES_PULL_SYS_TYPES_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/socket.h must be included by the external interface. */ +#define CARES_PULL_SYS_SOCKET_H 1 +#ifdef CARES_PULL_SYS_SOCKET_H +# include +#endif + +/* Integral data type used for ares_socklen_t. */ +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t + +/* Data type definition of ares_socklen_t. */ +typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; + +/* Integral data type used for ares_ssize_t. */ +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Data type definition of ares_ssize_t. */ +typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; + +#endif /* __CARES_BUILD_H */ diff --git a/contrib/c-ares-cmake/solaris/ares_config.h b/contrib/c-ares-cmake/solaris/ares_config.h new file mode 100644 index 00000000000..c4ac5e38966 --- /dev/null +++ b/contrib/c-ares-cmake/solaris/ares_config.h @@ -0,0 +1,503 @@ +/* src/lib/ares_config.h. Generated from ares_config.h.in by configure. */ +/* src/lib/ares_config.h.in. Generated from configure.ac by autoheader. */ + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define this if ares is built for a big endian system */ +/* #undef ARES_BIG_ENDIAN */ + +/* Defined for build that exposes internal static functions for testing. */ +/* #undef CARES_EXPOSE_STATICS */ + +/* a suitable file/device to read random data from */ +#define CARES_RANDOM_FILE "/dev/urandom" + +/* Defined for build with symbol hiding. */ +#define CARES_SYMBOL_HIDING 1 + +/* Definition to make a library symbol externally visible. */ +#define CARES_SYMBOL_SCOPE_EXTERN __attribute__ ((__visibility__ ("default"))) + +/* the signed version of size_t */ +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Use resolver library to configure cares */ +/* #undef CARES_USE_LIBRESOLV */ + +/* if a /etc/inet dir is being used */ +#define ETC_INET 1 + +/* Define to the type of arg 2 for gethostname. */ +#define GETHOSTNAME_TYPE_ARG2 int + +/* Define to the type qualifier of arg 1 for getnameinfo. */ +#define GETNAMEINFO_QUAL_ARG1 const + +/* Define to the type of arg 1 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG1 struct sockaddr * + +/* Define to the type of arg 2 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG2 socklen_t + +/* Define to the type of args 4 and 6 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG46 socklen_t + +/* Define to the type of arg 7 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG7 int + +/* Specifies the number of arguments to getservbyport_r */ +#define GETSERVBYPORT_R_ARGS 5 + +/* Specifies the size of the buffer to pass to getservbyport_r */ +#define GETSERVBYPORT_R_BUFSIZE 4096 + +/* Define to 1 if you have AF_INET6. */ +#define HAVE_AF_INET6 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_COMPAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bitncmp' function. */ +/* #undef HAVE_BITNCMP */ + +/* Define to 1 if bool is an available type. */ +#define HAVE_BOOL_T 1 + +/* Define to 1 if you have the clock_gettime function and monotonic timer. */ +#define HAVE_CLOCK_GETTIME_MONOTONIC 1 + +/* Define to 1 if you have the closesocket function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the CloseSocket camel case function. */ +/* #undef HAVE_CLOSESOCKET_CAMEL */ + +/* Define to 1 if you have the connect function. */ +#define HAVE_CONNECT 1 + +/* define if the compiler supports basic C++11 syntax */ +#define HAVE_CXX11 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the fcntl function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* Define to 1 if you have a working fcntl O_NONBLOCK function. */ +#define HAVE_FCNTL_O_NONBLOCK 1 + +/* Define to 1 if you have the freeaddrinfo function. */ +#define HAVE_FREEADDRINFO 1 + +/* Define to 1 if you have a working getaddrinfo function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if the getaddrinfo function is threadsafe. */ +#define HAVE_GETADDRINFO_THREADSAFE 1 + +/* Define to 1 if you have the getenv function. */ +#define HAVE_GETENV 1 + +/* Define to 1 if you have the gethostbyaddr function. */ +#define HAVE_GETHOSTBYADDR 1 + +/* Define to 1 if you have the gethostbyname function. */ +#define HAVE_GETHOSTBYNAME 1 + +/* Define to 1 if you have the gethostname function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the getnameinfo function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the getservbyport_r function. */ +#define HAVE_GETSERVBYPORT_R 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the `if_indextoname' function. */ +#define HAVE_IF_INDEXTONAME 1 + +/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */ +/* #undef HAVE_INET_NET_PTON */ + +/* Define to 1 if you have a IPv6 capable working inet_ntop function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have a IPv6 capable working inet_pton function. */ +#define HAVE_INET_PTON 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the ioctl function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the ioctlsocket function. */ +/* #undef HAVE_IOCTLSOCKET */ + +/* Define to 1 if you have the IoctlSocket camel case function. */ +/* #undef HAVE_IOCTLSOCKET_CAMEL */ + +/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function. + */ +/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */ + +/* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* #undef HAVE_IOCTLSOCKET_FIONBIO */ + +/* Define to 1 if you have a working ioctl FIONBIO function. */ +/* #undef HAVE_IOCTL_FIONBIO */ + +/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */ +/* #undef HAVE_IOCTL_SIOCGIFADDR */ + +/* Define to 1 if you have the `resolve' library (-lresolve). */ +/* #undef HAVE_LIBRESOLVE */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if your compiler supports LL */ +#define HAVE_LL 1 + +/* Define to 1 if the compiler supports the 'long long' data type. */ +#define HAVE_LONGLONG 1 + +/* Define to 1 if you have the malloc.h header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the memory.h header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the MSG_NOSIGNAL flag. */ +#define HAVE_MSG_NOSIGNAL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H 1 + +/* Define to 1 if you have PF_INET6. */ +#define HAVE_PF_INET6 1 + +/* Define to 1 if you have the recv function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the recvfrom function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the send function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the setsockopt function. */ +#define HAVE_SETSOCKOPT 1 + +/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */ +/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H 1 + +/* Define to 1 if sig_atomic_t is an available typedef. */ +#define HAVE_SIG_ATOMIC_T 1 + +/* Define to 1 if sig_atomic_t is already defined as volatile. */ +/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */ + +/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */ +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID 1 + +/* Define to 1 if you have the socket function. */ +#define HAVE_SOCKET 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SOCKET_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDBOOL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the strcasecmp function. */ +#define HAVE_STRCASECMP 1 + +/* Define to 1 if you have the strcmpi function. */ +/* #undef HAVE_STRCMPI */ + +/* Define to 1 if you have the strdup function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the stricmp function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the strncasecmp function. */ +#define HAVE_STRNCASECMP 1 + +/* Define to 1 if you have the strncmpi function. */ +/* #undef HAVE_STRNCMPI */ + +/* Define to 1 if you have the strnicmp function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STROPTS_H 1 + +/* Define to 1 if you have struct addrinfo. */ +#define HAVE_STRUCT_ADDRINFO 1 + +/* Define to 1 if you have struct in6_addr. */ +#define HAVE_STRUCT_IN6_ADDR 1 + +/* Define to 1 if you have struct sockaddr_in6. */ +#define HAVE_STRUCT_SOCKADDR_IN6 1 + +/* if struct sockaddr_storage is defined */ +#define HAVE_STRUCT_SOCKADDR_STORAGE 1 + +/* Define to 1 if you have the timeval struct. */ +#define HAVE_STRUCT_TIMEVAL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the windows.h header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the winsock2.h header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the winsock.h header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the writev function. */ +#define HAVE_WRITEV 1 + +/* Define to 1 if you have the ws2tcpip.h header file. */ +/* #undef HAVE_WS2TCPIP_H */ + +/* Define if __system_property_get exists. */ +/* #undef HAVE___SYSTEM_PROPERTY_GET */ + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* Define to 1 if you need the malloc.h header file even with stdlib.h */ +/* #undef NEED_MALLOC_H */ + +/* Define to 1 if you need the memory.h header file even with stdlib.h */ +/* #undef NEED_MEMORY_H */ + +/* Define to 1 if _REENTRANT preprocessor symbol must be defined. */ +#define NEED_REENTRANT 1 + +/* Define to 1 if _THREAD_SAFE preprocessor symbol must be defined. */ +/* #undef NEED_THREAD_SAFE */ + +/* cpu-machine-OS */ +#define OS "x86_64-pc-solaris2.11" + +/* Name of package */ +#define PACKAGE "c-ares" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "c-ares mailing list: http://lists.haxx.se/listinfo/c-ares" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "c-ares" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "c-ares 1.18.1" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "c-ares" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "1.18.1" + +/* Define to the type qualifier pointed by arg 5 for recvfrom. */ +#define RECVFROM_QUAL_ARG5 + +/* Define to the type of arg 1 for recvfrom. */ +#define RECVFROM_TYPE_ARG1 int + +/* Define to the type pointed by arg 2 for recvfrom. */ +#define RECVFROM_TYPE_ARG2 void + +/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG2_IS_VOID 1 + +/* Define to the type of arg 3 for recvfrom. */ +#define RECVFROM_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recvfrom. */ +#define RECVFROM_TYPE_ARG4 int + +/* Define to the type pointed by arg 5 for recvfrom. */ +#define RECVFROM_TYPE_ARG5 struct sockaddr + +/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */ +/* #undef RECVFROM_TYPE_ARG5_IS_VOID */ + +/* Define to the type pointed by arg 6 for recvfrom. */ +#define RECVFROM_TYPE_ARG6 void + +/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG6_IS_VOID 1 + +/* Define to the function return type for recvfrom. */ +#define RECVFROM_TYPE_RETV ssize_t + +/* Define to the type of arg 1 for recv. */ +#define RECV_TYPE_ARG1 int + +/* Define to the type of arg 2 for recv. */ +#define RECV_TYPE_ARG2 void * + +/* Define to the type of arg 3 for recv. */ +#define RECV_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recv. */ +#define RECV_TYPE_ARG4 int + +/* Define to the function return type for recv. */ +#define RECV_TYPE_RETV ssize_t + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type qualifier of arg 2 for send. */ +#define SEND_QUAL_ARG2 const + +/* Define to the type of arg 1 for send. */ +#define SEND_TYPE_ARG1 int + +/* Define to the type of arg 2 for send. */ +#define SEND_TYPE_ARG2 void * + +/* Define to the type of arg 3 for send. */ +#define SEND_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for send. */ +#define SEND_TYPE_ARG4 int + +/* Define to the function return type for send. */ +#define SEND_TYPE_RETV ssize_t + +/* Define to 1 if all of the C90 standard headers exist (not just the ones + required in a freestanding environment). This macro is provided for + backward compatibility; new code need not use it. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . This + macro is obsolete. */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to disable non-blocking sockets. */ +/* #undef USE_BLOCKING_SOCKETS */ + +/* Version number of package */ +#define VERSION "1.18.1" + +/* Define to avoid automatic inclusion of winsock.h */ +/* #undef WIN32_LEAN_AND_MEAN */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to 1 if OS is AIX. */ +#ifndef _ALL_SOURCE +/* # undef _ALL_SOURCE */ +#endif + +/* Number of bits in a file offset, on hosts where this is settable. */ +/* #undef _FILE_OFFSET_BITS */ + +/* Define for large files, on AIX-style hosts. */ +/* #undef _LARGE_FILES */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Type to use in place of in_addr_t when system does not provide it. */ +/* #undef in_addr_t */ + +/* Define to `unsigned int' if does not define. */ +/* #undef size_t */ diff --git a/contrib/cppkafka b/contrib/cppkafka index 5a119f689f8..9c5ea0e3324 160000 --- a/contrib/cppkafka +++ b/contrib/cppkafka @@ -1 +1 @@ -Subproject commit 5a119f689f8a4d90d10a9635e7ee2bee5c127de1 +Subproject commit 9c5ea0e332486961e612deacc6e3f0c1874c688d diff --git a/contrib/curl-cmake/curl_config.h b/contrib/curl-cmake/curl_config.h index a38aa60fe6d..4d4c2972f57 100644 --- a/contrib/curl-cmake/curl_config.h +++ b/contrib/curl-cmake/curl_config.h @@ -51,3 +51,8 @@ #define USE_OPENSSL #define USE_THREADS_POSIX #define USE_ARES + +#ifdef __illumos__ +#define HAVE_POSIX_STRERROR_R 1 +#define HAVE_STRERROR_R 1 +#endif diff --git a/contrib/libssh-cmake/CMakeLists.txt b/contrib/libssh-cmake/CMakeLists.txt index 7b589718140..ecd1fccb800 100644 --- a/contrib/libssh-cmake/CMakeLists.txt +++ b/contrib/libssh-cmake/CMakeLists.txt @@ -1,26 +1,18 @@ -option (ENABLE_SSH "Enable support for SSH keys and protocol" ${ENABLE_LIBRARIES}) +option (ENABLE_SSH "Enable support for libssh" ${ENABLE_LIBRARIES}) if (NOT ENABLE_SSH) - message(STATUS "Not using SSH") + message(STATUS "Not using libssh") return() endif() +# CMake variables needed by libssh_version.h.cmake, update them when you update libssh +set(libssh_VERSION_MAJOR 0) +set(libssh_VERSION_MINOR 9) +set(libssh_VERSION_PATCH 8) + set(LIB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libssh") set(LIB_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/libssh") -# Set CMake variables which are used in libssh_version.h.cmake -project(libssh VERSION 0.9.8 LANGUAGES C) - -set(LIBRARY_VERSION "4.8.8") -set(LIBRARY_SOVERSION "4") - -set(CMAKE_THREAD_PREFER_PTHREADS ON) -set(THREADS_PREFER_PTHREAD_FLAG ON) - -set(WITH_ZLIB OFF) -set(WITH_SYMBOL_VERSIONING OFF) -set(WITH_SERVER ON) - set(libssh_SRCS ${LIB_SOURCE_DIR}/src/agent.c ${LIB_SOURCE_DIR}/src/auth.c @@ -28,15 +20,21 @@ set(libssh_SRCS ${LIB_SOURCE_DIR}/src/bignum.c ${LIB_SOURCE_DIR}/src/buffer.c ${LIB_SOURCE_DIR}/src/callbacks.c + ${LIB_SOURCE_DIR}/src/chachapoly.c ${LIB_SOURCE_DIR}/src/channels.c ${LIB_SOURCE_DIR}/src/client.c ${LIB_SOURCE_DIR}/src/config.c + ${LIB_SOURCE_DIR}/src/config_parser.c ${LIB_SOURCE_DIR}/src/connect.c ${LIB_SOURCE_DIR}/src/connector.c ${LIB_SOURCE_DIR}/src/curve25519.c ${LIB_SOURCE_DIR}/src/dh.c ${LIB_SOURCE_DIR}/src/ecdh.c ${LIB_SOURCE_DIR}/src/error.c + ${LIB_SOURCE_DIR}/src/external/bcrypt_pbkdf.c + ${LIB_SOURCE_DIR}/src/external/blowfish.c + ${LIB_SOURCE_DIR}/src/external/chacha.c + ${LIB_SOURCE_DIR}/src/external/poly1305.c ${LIB_SOURCE_DIR}/src/getpass.c ${LIB_SOURCE_DIR}/src/init.c ${LIB_SOURCE_DIR}/src/kdf.c @@ -55,37 +53,32 @@ set(libssh_SRCS ${LIB_SOURCE_DIR}/src/pcap.c ${LIB_SOURCE_DIR}/src/pki.c ${LIB_SOURCE_DIR}/src/pki_container_openssh.c + ${LIB_SOURCE_DIR}/src/pki_ed25519_common.c ${LIB_SOURCE_DIR}/src/poll.c - ${LIB_SOURCE_DIR}/src/session.c ${LIB_SOURCE_DIR}/src/scp.c + ${LIB_SOURCE_DIR}/src/session.c ${LIB_SOURCE_DIR}/src/socket.c ${LIB_SOURCE_DIR}/src/string.c ${LIB_SOURCE_DIR}/src/threads.c - ${LIB_SOURCE_DIR}/src/wrapper.c - ${LIB_SOURCE_DIR}/src/external/bcrypt_pbkdf.c - ${LIB_SOURCE_DIR}/src/external/blowfish.c - ${LIB_SOURCE_DIR}/src/external/chacha.c - ${LIB_SOURCE_DIR}/src/external/poly1305.c - ${LIB_SOURCE_DIR}/src/chachapoly.c - ${LIB_SOURCE_DIR}/src/config_parser.c ${LIB_SOURCE_DIR}/src/token.c - ${LIB_SOURCE_DIR}/src/pki_ed25519_common.c + ${LIB_SOURCE_DIR}/src/wrapper.c + # some files of libssh/src/ are missing - why? ${LIB_SOURCE_DIR}/src/threads/noop.c ${LIB_SOURCE_DIR}/src/threads/pthread.c + # files missing - why? # LIBCRYPT specific - ${libssh_SRCS} - ${LIB_SOURCE_DIR}/src/threads/libcrypto.c - ${LIB_SOURCE_DIR}/src/pki_crypto.c + ${LIB_SOURCE_DIR}/src/dh_crypto.c ${LIB_SOURCE_DIR}/src/ecdh_crypto.c ${LIB_SOURCE_DIR}/src/libcrypto.c - ${LIB_SOURCE_DIR}/src/dh_crypto.c + ${LIB_SOURCE_DIR}/src/pki_crypto.c + ${LIB_SOURCE_DIR}/src/threads/libcrypto.c - ${LIB_SOURCE_DIR}/src/options.c - ${LIB_SOURCE_DIR}/src/server.c ${LIB_SOURCE_DIR}/src/bind.c ${LIB_SOURCE_DIR}/src/bind_config.c + ${LIB_SOURCE_DIR}/src/options.c + ${LIB_SOURCE_DIR}/src/server.c ) if (NOT (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC)) @@ -94,7 +87,7 @@ endif() configure_file(${LIB_SOURCE_DIR}/include/libssh/libssh_version.h.cmake ${LIB_BINARY_DIR}/include/libssh/libssh_version.h @ONLY) -add_library(_ssh STATIC ${libssh_SRCS}) +add_library(_ssh ${libssh_SRCS}) add_library(ch_contrib::ssh ALIAS _ssh) target_link_libraries(_ssh PRIVATE OpenSSL::Crypto) diff --git a/contrib/nuraft-cmake/CMakeLists.txt b/contrib/nuraft-cmake/CMakeLists.txt index eaca00566d6..736e91e359d 100644 --- a/contrib/nuraft-cmake/CMakeLists.txt +++ b/contrib/nuraft-cmake/CMakeLists.txt @@ -32,6 +32,7 @@ set(SRCS "${LIBRARY_DIR}/src/handle_custom_notification.cxx" "${LIBRARY_DIR}/src/handle_vote.cxx" "${LIBRARY_DIR}/src/launcher.cxx" + "${LIBRARY_DIR}/src/log_entry.cxx" "${LIBRARY_DIR}/src/srv_config.cxx" "${LIBRARY_DIR}/src/snapshot_sync_req.cxx" "${LIBRARY_DIR}/src/snapshot_sync_ctx.cxx" @@ -50,6 +51,12 @@ else() target_compile_definitions(_nuraft PRIVATE USE_BOOST_ASIO=1 BOOST_ASIO_STANDALONE=1) endif() +target_link_libraries (_nuraft PRIVATE clickhouse_common_io) +# We must have it PUBLIC here because some headers which depend on it directly +# included in clickhouse +target_compile_definitions(_nuraft PUBLIC USE_CLICKHOUSE_THREADS=1) +MESSAGE(STATUS "Will use clickhouse threads for NuRaft") + target_include_directories (_nuraft SYSTEM PRIVATE "${LIBRARY_DIR}/include/libnuraft") # for some reason include "asio.h" directly without "boost/" prefix. target_include_directories (_nuraft SYSTEM PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") diff --git a/contrib/xxHash b/contrib/xxHash index 3078dc6039f..bbb27a5efb8 160000 --- a/contrib/xxHash +++ b/contrib/xxHash @@ -1 +1 @@ -Subproject commit 3078dc6039f8c0bffcb1904f81cfe6b2c3209435 +Subproject commit bbb27a5efb85b92a0486cf361a8635715a53f6ba diff --git a/contrib/xxHash-cmake/CMakeLists.txt b/contrib/xxHash-cmake/CMakeLists.txt index 314094e9523..bd7192ae944 100644 --- a/contrib/xxHash-cmake/CMakeLists.txt +++ b/contrib/xxHash-cmake/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(xxHash ${SRCS}) target_include_directories(xxHash SYSTEM BEFORE INTERFACE "${LIBRARY_DIR}") # XXH_INLINE_ALL - Make all functions inline, with implementations being directly included within xxhash.h. Inlining functions is beneficial for speed on small keys. -# https://github.com/Cyan4973/xxHash/tree/v0.8.1#build-modifiers +# https://github.com/Cyan4973/xxHash/tree/v0.8.2#build-modifiers target_compile_definitions(xxHash PUBLIC XXH_INLINE_ALL) add_library(ch_contrib::xxHash ALIAS xxHash) diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 17eee6d4287..346868e19c4 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.2.2.71" +ARG VERSION="24.3.2.23" ARG PACKAGES="clickhouse-keeper" ARG DIRECT_DOWNLOAD_URLS="" @@ -44,7 +44,10 @@ ARG DIRECT_DOWNLOAD_URLS="" # We do that in advance at the begining of Dockerfile before any packages will be # installed to prevent picking those uid / gid by some unrelated software. # The same uid / gid (101) is used both for alpine and ubuntu. - +ARG DEFAULT_UID="101" +ARG DEFAULT_GID="101" +RUN addgroup -S -g "${DEFAULT_GID}" clickhouse && \ + adduser -S -h "/var/lib/clickhouse" -s /bin/bash -G clickhouse -g "ClickHouse keeper" -u "${DEFAULT_UID}" clickhouse ARG TARGETARCH RUN arch=${TARGETARCH:-amd64} \ @@ -71,20 +74,21 @@ RUN arch=${TARGETARCH:-amd64} \ fi \ ; done \ && rm /tmp/*.tgz /install -r \ - && addgroup -S -g 101 clickhouse \ - && adduser -S -h /var/lib/clickhouse -s /bin/bash -G clickhouse -g "ClickHouse keeper" -u 101 clickhouse \ - && mkdir -p /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper \ - && chown clickhouse:clickhouse /var/lib/clickhouse \ - && chown root:clickhouse /var/log/clickhouse-keeper \ && chmod +x /entrypoint.sh \ && apk add --no-cache su-exec bash tzdata \ && cp /usr/share/zoneinfo/UTC /etc/localtime \ - && echo "UTC" > /etc/timezone \ - && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper + && echo "UTC" > /etc/timezone +ARG DEFAULT_CONFIG_DIR="/etc/clickhouse-keeper" +ARG DEFAULT_DATA_DIR="/var/lib/clickhouse-keeper" +ARG DEFAULT_LOG_DIR="/var/log/clickhouse-keeper" +RUN mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" \ + && chown clickhouse:clickhouse "${DEFAULT_DATA_DIR}" \ + && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ + && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" +# /var/lib/clickhouse is necessary due to the current default configuration for Keeper +VOLUME "${DEFAULT_DATA_DIR}" /var/lib/clickhouse EXPOSE 2181 10181 44444 9181 -VOLUME /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper - ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/keeper/entrypoint.sh b/docker/keeper/entrypoint.sh index 939cd941aeb..1390ad9ce74 100644 --- a/docker/keeper/entrypoint.sh +++ b/docker/keeper/entrypoint.sh @@ -80,7 +80,7 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then # so the container can't be finished by ctrl+c export CLICKHOUSE_WATCHDOG_ENABLE - cd /var/lib/clickhouse + cd "${DATA_DIR}" # There is a config file. It is already tested with gosu (if it is readably by keeper user) if [ -f "$KEEPER_CONFIG" ]; then diff --git a/docker/packager/binary-builder/Dockerfile b/docker/packager/binary-builder/Dockerfile index c9442accd7e..73ec4275f12 100644 --- a/docker/packager/binary-builder/Dockerfile +++ b/docker/packager/binary-builder/Dockerfile @@ -61,7 +61,7 @@ RUN arch=${TARGETARCH:-amd64} \ && rm /tmp/nfpm.deb ARG GO_VERSION=1.19.10 -# We need go for clickhouse-diagnostics +# We needed go for clickhouse-diagnostics (it is not used anymore) RUN arch=${TARGETARCH:-amd64} \ && curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \ && tar -xzf /tmp/go.tgz -C /usr/local/ \ diff --git a/docker/packager/binary-builder/build.sh b/docker/packager/binary-builder/build.sh index b63643419fe..032aceb0af3 100755 --- a/docker/packager/binary-builder/build.sh +++ b/docker/packager/binary-builder/build.sh @@ -36,22 +36,6 @@ rm -f CMakeCache.txt if [ -n "$MAKE_DEB" ]; then rm -rf /build/packages/root - # NOTE: this is for backward compatibility with previous releases, - # that does not diagnostics tool (only script). - if [ -d /build/programs/diagnostics ]; then - if [ -z "$SANITIZER" ]; then - # We need to check if clickhouse-diagnostics is fine and build it - ( - cd /build/programs/diagnostics - make test-no-docker - GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build - mv clickhouse-diagnostics .. - ) - else - echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics - chmod +x /build/programs/clickhouse-diagnostics - fi - fi fi @@ -121,8 +105,6 @@ if [ -n "$MAKE_DEB" ]; then # No quotes because I want it to expand to nothing if empty. # shellcheck disable=SC2086 DESTDIR=/build/packages/root ninja $NINJA_FLAGS programs/install - cp /build/programs/clickhouse-diagnostics /build/packages/root/usr/bin - cp /build/programs/clickhouse-diagnostics /output bash -x /build/packages/build fi diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index bd5fa313adc..36f09c092f8 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.2.2.71" +ARG VERSION="24.3.2.23" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" ARG DIRECT_DOWNLOAD_URLS="" @@ -42,6 +42,10 @@ ARG DIRECT_DOWNLOAD_URLS="" # We do that in advance at the begining of Dockerfile before any packages will be # installed to prevent picking those uid / gid by some unrelated software. # The same uid / gid (101) is used both for alpine and ubuntu. +ARG DEFAULT_UID="101" +ARG DEFAULT_GID="101" +RUN addgroup -S -g "${DEFAULT_GID}" clickhouse && \ + adduser -S -h "/var/lib/clickhouse" -s /bin/bash -G clickhouse -g "ClickHouse server" -u "${DEFAULT_UID}" clickhouse RUN arch=${TARGETARCH:-amd64} \ && cd /tmp \ @@ -66,23 +70,30 @@ RUN arch=${TARGETARCH:-amd64} \ fi \ ; done \ && rm /tmp/*.tgz /install -r \ - && addgroup -S -g 101 clickhouse \ - && adduser -S -h /var/lib/clickhouse -s /bin/bash -G clickhouse -g "ClickHouse server" -u 101 clickhouse \ - && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server/config.d /etc/clickhouse-server/users.d /etc/clickhouse-client /docker-entrypoint-initdb.d \ - && chown clickhouse:clickhouse /var/lib/clickhouse \ - && chown root:clickhouse /var/log/clickhouse-server \ && chmod +x /entrypoint.sh \ && apk add --no-cache bash tzdata \ && cp /usr/share/zoneinfo/UTC /etc/localtime \ - && echo "UTC" > /etc/timezone \ - && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client + && echo "UTC" > /etc/timezone -# we need to allow "others" access to clickhouse folder, because docker container -# can be started with arbitrary uid (openshift usecase) +ARG DEFAULT_CLIENT_CONFIG_DIR="/etc/clickhouse-client" +ARG DEFAULT_SERVER_CONFIG_DIR="/etc/clickhouse-server" +ARG DEFAULT_DATA_DIR="/var/lib/clickhouse" +ARG DEFAULT_LOG_DIR="/var/log/clickhouse-server" +# we need to allow "others" access to ClickHouse folders, because docker containers +# can be started with arbitrary uids (OpenShift usecase) +RUN mkdir -p \ + "${DEFAULT_DATA_DIR}" \ + "${DEFAULT_LOG_DIR}" \ + "${DEFAULT_CLIENT_CONFIG_DIR}" \ + "${DEFAULT_SERVER_CONFIG_DIR}/config.d" \ + "${DEFAULT_SERVER_CONFIG_DIR}/users.d" \ + /docker-entrypoint-initdb.d \ + && chown clickhouse:clickhouse "${DEFAULT_DATA_DIR}" \ + && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ + && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CLIENT_CONFIG_DIR}" "${DEFAULT_SERVER_CONFIG_DIR}" + +VOLUME "${DEFAULT_DATA_DIR}" EXPOSE 9000 8123 9009 -VOLUME /var/lib/clickhouse \ - /var/log/clickhouse-server - ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 256dcdc029f..531a50efe96 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -27,7 +27,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="24.2.2.71" +ARG VERSION="24.3.2.23" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 55f4d379005..76661a5b51c 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -181,7 +181,15 @@ function fuzz -- --path db \ --logger.console=0 \ --logger.log=server.log 2>&1 | tee -a stderr.log >> server.log 2>&1 & - server_pid=$(pidof clickhouse-server) + for _ in {1..30} + do + if clickhouse-client --query "select 1" + then + break + fi + sleep 1 + done + server_pid=$(cat /var/run/clickhouse-server/clickhouse-server.pid) kill -0 $server_pid diff --git a/docker/test/stateful/create.sql b/docker/test/stateful/create.sql index f3e869402d9..14e592ab5e3 100644 --- a/docker/test/stateful/create.sql +++ b/docker/test/stateful/create.sql @@ -138,7 +138,7 @@ ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) -SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/filesystem_caches/', max_size = '4G', +SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/filesystem_caches/stateful/', max_size = '4G', disk = disk(type = web, endpoint = 'https://clickhouse-datasets-web.s3.us-east-1.amazonaws.com/')); ATTACH TABLE datasets.visits_v1 UUID '5131f834-711f-4168-98a5-968b691a104b' @@ -329,5 +329,5 @@ ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) SAMPLE BY intHash32(UserID) -SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/filesystem_caches/', max_size = '4G', +SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/filesystem_caches/stateful/', max_size = '4G', disk = disk(type = web, endpoint = 'https://clickhouse-datasets-web.s3.us-east-1.amazonaws.com/')); diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 98da5988ad5..c2e9fdfe41d 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -25,7 +25,7 @@ azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml cache_policy="" -if [ $(( $(date +%-d) % 2 )) -eq 1 ]; then +if [ $(($RANDOM%2)) -eq 1 ]; then cache_policy="SLRU" else cache_policy="LRU" diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index bac9d8df7a9..271f30d187b 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -16,6 +16,8 @@ ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone dpkg -i package_folder/clickhouse-common-static_*.deb dpkg -i package_folder/clickhouse-common-static-dbg_*.deb +dpkg -i package_folder/clickhouse-odbc-bridge_*.deb +dpkg -i package_folder/clickhouse-library-bridge_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb @@ -41,6 +43,8 @@ source /utils.lib if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then echo "Azure is disabled" +elif [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + echo "Azure is disabled" else azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & fi @@ -137,6 +141,32 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited) fi +if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \ + | sed "s|/var/lib/clickhouse/filesystem_caches/|/var/lib/clickhouse/filesystem_caches_1/|" \ + > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp + mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml + + sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \ + | sed "s|/var/lib/clickhouse/filesystem_caches/|/var/lib/clickhouse/filesystem_caches_1/|" \ + > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp + mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml + + mkdir -p /var/run/clickhouse-server1 + sudo chown clickhouse:clickhouse /var/run/clickhouse-server1 + sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \ + --pid-file /var/run/clickhouse-server1/clickhouse-server.pid \ + -- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \ + --logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \ + --tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \ + --mysql_port 19004 --postgresql_port 19005 \ + --keeper_server.tcp_port 19181 --keeper_server.server_id 2 \ + --prometheus.port 19988 \ + --macros.replica r2 # It doesn't work :( + + MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000)) # min(MAX_RUN_TIME, 2.5 hours) + MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited) +fi # Wait for the server to start, but not for too long. for _ in {1..100} @@ -183,6 +213,10 @@ function run_tests() ADDITIONAL_OPTIONS+=('--s3-storage') fi + if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + ADDITIONAL_OPTIONS+=('--shared-catalog') + fi + if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then ADDITIONAL_OPTIONS+=('--replicated-database') # Too many tests fail for DatabaseReplicated in parallel. @@ -257,10 +291,16 @@ do echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then - err=$( { clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) + err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 - err=$( { clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 ) + err=$( { clickhouse-client --port 29000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 ) + echo "$err" + [[ "0" != "${#err}" ]] && failed_to_save_logs=1 + fi + + if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 fi @@ -275,6 +315,10 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||: fi +if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||: +fi + rg -Fa "" /var/log/clickhouse-server/clickhouse-server.log ||: rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||: zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst & @@ -302,6 +346,10 @@ if [ $failed_to_save_logs -ne 0 ]; then clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||: clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst ||: fi + + if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||: + fi done fi @@ -341,3 +389,10 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||: tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||: fi + +if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then + rg -Fa "" /var/log/clickhouse-server/clickhouse-server1.log ||: + zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||: + mv /var/log/clickhouse-server/stderr1.log /test_output/ ||: + tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||: +fi diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index ea7e3aece1d..6c6caf872e9 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -72,7 +72,7 @@ mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/c # Randomize cache policies. cache_policy="" -if [ $(( $(date +%-d) % 2 )) -eq 1 ]; then +if [ $(($RANDOM%2)) -eq 1 ]; then cache_policy="SLRU" else cache_policy="LRU" @@ -87,6 +87,25 @@ if [ "$cache_policy" = "SLRU" ]; then mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml fi +# Disable experimental WINDOW VIEW tests for stress tests, since they may be +# created with old analyzer and then, after server restart it will refuse to +# start. +# FIXME: remove once the support for WINDOW VIEW will be implemented in analyzer. +sudo cat /etc/clickhouse-server/users.d/stress_tests_overrides.xml < + + + false + + + + + + + + +EOL + start_server clickhouse-client --query "SHOW TABLES FROM datasets" diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index b4ffcfb597c..5d53d03606f 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -8,20 +8,22 @@ ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ - aspell \ - curl \ - git \ - file \ - libxml2-utils \ - moreutils \ - python3-fuzzywuzzy \ - python3-pip \ - yamllint \ - locales \ - && pip3 install black==23.12.0 boto3 codespell==2.2.1 mypy==1.8.0 PyGithub unidiff pylint==3.1.0 \ - requests types-requests \ + aspell \ + curl \ + git \ + file \ + libxml2-utils \ + moreutils \ + python3-fuzzywuzzy \ + python3-pip \ + yamllint \ + locales \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* + +# python-magic is the same version as in Ubuntu 22.04 +RUN pip3 install black==23.12.0 boto3 codespell==2.2.1 mypy==1.8.0 PyGithub unidiff pylint==3.1.0 \ + python-magic==0.4.24 requests types-requests \ && rm -rf /root/.cache/pip RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8 diff --git a/docs/changelogs/v23.12.6.19-stable.md b/docs/changelogs/v23.12.6.19-stable.md new file mode 100644 index 00000000000..4659532d3de --- /dev/null +++ b/docs/changelogs/v23.12.6.19-stable.md @@ -0,0 +1,24 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v23.12.6.19-stable (40080a3c2a4) FIXME as compared to v23.12.5.81-stable (a0fbe3ae813) + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Improve isolation of query cache entries under re-created users or role switches [#58611](https://github.com/ClickHouse/ClickHouse/pull/58611) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix possible incorrect result of aggregate function `uniqExact` [#61257](https://github.com/ClickHouse/ClickHouse/pull/61257) ([Anton Popov](https://github.com/CurtizJ)). +* Fix consecutive keys optimization for nullable keys [#61393](https://github.com/ClickHouse/ClickHouse/pull/61393) ([Anton Popov](https://github.com/CurtizJ)). +* Fix string search with const position [#61547](https://github.com/ClickHouse/ClickHouse/pull/61547) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix crash in `multiSearchAllPositionsCaseInsensitiveUTF8` for incorrect UTF-8 [#61749](https://github.com/ClickHouse/ClickHouse/pull/61749) ([pufit](https://github.com/pufit)). + +#### CI Fix or Improvement (changelog entry is not required) + +* Backported in [#61429](https://github.com/ClickHouse/ClickHouse/issues/61429):. [#61374](https://github.com/ClickHouse/ClickHouse/pull/61374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#61486](https://github.com/ClickHouse/ClickHouse/issues/61486): ... [#61441](https://github.com/ClickHouse/ClickHouse/pull/61441) ([Max K.](https://github.com/maxknv)). +* Backported in [#61641](https://github.com/ClickHouse/ClickHouse/issues/61641):. [#61592](https://github.com/ClickHouse/ClickHouse/pull/61592) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#61811](https://github.com/ClickHouse/ClickHouse/issues/61811): ![Screenshot_20240323_025055](https://github.com/ClickHouse/ClickHouse/assets/18581488/ccaab212-a1d3-4dfb-8d56-b1991760b6bf). [#61801](https://github.com/ClickHouse/ClickHouse/pull/61801) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v23.3.22.3-lts.md b/docs/changelogs/v23.3.22.3-lts.md new file mode 100644 index 00000000000..2900480e12d --- /dev/null +++ b/docs/changelogs/v23.3.22.3-lts.md @@ -0,0 +1,13 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v23.3.22.3-lts (04075bf96a1) FIXME as compared to v23.3.21.26-lts (d9672a3731f) + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix crash in `multiSearchAllPositionsCaseInsensitiveUTF8` for incorrect UTF-8 [#61749](https://github.com/ClickHouse/ClickHouse/pull/61749) ([pufit](https://github.com/pufit)). + diff --git a/docs/changelogs/v23.8.12.13-lts.md b/docs/changelogs/v23.8.12.13-lts.md new file mode 100644 index 00000000000..dbb36fdc00e --- /dev/null +++ b/docs/changelogs/v23.8.12.13-lts.md @@ -0,0 +1,20 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v23.8.12.13-lts (bdbd0d87e5d) FIXME as compared to v23.8.11.28-lts (31879d2ab4c) + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Improve isolation of query cache entries under re-created users or role switches [#58611](https://github.com/ClickHouse/ClickHouse/pull/58611) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix string search with const position [#61547](https://github.com/ClickHouse/ClickHouse/pull/61547) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix crash in `multiSearchAllPositionsCaseInsensitiveUTF8` for incorrect UTF-8 [#61749](https://github.com/ClickHouse/ClickHouse/pull/61749) ([pufit](https://github.com/pufit)). + +#### CI Fix or Improvement (changelog entry is not required) + +* Backported in [#61428](https://github.com/ClickHouse/ClickHouse/issues/61428):. [#61374](https://github.com/ClickHouse/ClickHouse/pull/61374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#61484](https://github.com/ClickHouse/ClickHouse/issues/61484): ... [#61441](https://github.com/ClickHouse/ClickHouse/pull/61441) ([Max K.](https://github.com/maxknv)). + diff --git a/docs/changelogs/v24.1.8.22-stable.md b/docs/changelogs/v24.1.8.22-stable.md new file mode 100644 index 00000000000..f780de41c40 --- /dev/null +++ b/docs/changelogs/v24.1.8.22-stable.md @@ -0,0 +1,32 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.1.8.22-stable (7fb8f96d3da) FIXME as compared to v24.1.7.18-stable (90925babd78) + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix possible incorrect result of aggregate function `uniqExact` [#61257](https://github.com/ClickHouse/ClickHouse/pull/61257) ([Anton Popov](https://github.com/CurtizJ)). +* Fix consecutive keys optimization for nullable keys [#61393](https://github.com/ClickHouse/ClickHouse/pull/61393) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bug when reading system.parts using UUID (issue 61220). [#61479](https://github.com/ClickHouse/ClickHouse/pull/61479) ([Dan Wu](https://github.com/wudanzy)). +* Fix client `-s` argument [#61530](https://github.com/ClickHouse/ClickHouse/pull/61530) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix string search with const position [#61547](https://github.com/ClickHouse/ClickHouse/pull/61547) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix crash in `multiSearchAllPositionsCaseInsensitiveUTF8` for incorrect UTF-8 [#61749](https://github.com/ClickHouse/ClickHouse/pull/61749) ([pufit](https://github.com/pufit)). + +#### CI Fix or Improvement (changelog entry is not required) + +* Backported in [#61431](https://github.com/ClickHouse/ClickHouse/issues/61431):. [#61374](https://github.com/ClickHouse/ClickHouse/pull/61374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#61488](https://github.com/ClickHouse/ClickHouse/issues/61488): ... [#61441](https://github.com/ClickHouse/ClickHouse/pull/61441) ([Max K.](https://github.com/maxknv)). +* Backported in [#61642](https://github.com/ClickHouse/ClickHouse/issues/61642):. [#61592](https://github.com/ClickHouse/ClickHouse/pull/61592) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Backport [#61479](https://github.com/ClickHouse/ClickHouse/issues/61479) to 24.1: Fix bug when reading system.parts using UUID (issue 61220)."'. [#61775](https://github.com/ClickHouse/ClickHouse/pull/61775) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Speed up cctools building [#61011](https://github.com/ClickHouse/ClickHouse/pull/61011) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v24.3.1.2672-lts.md b/docs/changelogs/v24.3.1.2672-lts.md new file mode 100644 index 00000000000..e5d008680a8 --- /dev/null +++ b/docs/changelogs/v24.3.1.2672-lts.md @@ -0,0 +1,537 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.3.1.2672-lts (2c5c589a882) FIXME as compared to v24.2.1.2248-stable (891689a4150) + +#### Backward Incompatible Change +* Don't allow to set max_parallel_replicas to 0 as it doesn't make sense. Setting it to 0 could lead to unexpected logical errors. Closes [#60140](https://github.com/ClickHouse/ClickHouse/issues/60140). [#60430](https://github.com/ClickHouse/ClickHouse/pull/60430) ([Kruglov Pavel](https://github.com/Avogar)). +* Change the column name from `duration_ms` to `duration_microseconds` in the `system.zookeeper` table to reflect the reality that the duration is in the microsecond resolution. [#60774](https://github.com/ClickHouse/ClickHouse/pull/60774) ([Duc Canh Le](https://github.com/canhld94)). +* Reject incoming INSERT queries in case when query-level settings `async_insert` and `deduplicate_blocks_in_dependent_materialized_views` are enabled together. This behaviour is controlled by a setting `throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert` and enabled by default. This is a continuation of https://github.com/ClickHouse/ClickHouse/pull/59699 needed to unblock https://github.com/ClickHouse/ClickHouse/pull/59915. [#60888](https://github.com/ClickHouse/ClickHouse/pull/60888) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Utility `clickhouse-copier` is moved to a separate repository on GitHub: https://github.com/ClickHouse/copier. It is no longer included in the bundle but is still available as a separate download. This closes: [#60734](https://github.com/ClickHouse/ClickHouse/issues/60734) This closes: [#60540](https://github.com/ClickHouse/ClickHouse/issues/60540) This closes: [#60250](https://github.com/ClickHouse/ClickHouse/issues/60250) This closes: [#52917](https://github.com/ClickHouse/ClickHouse/issues/52917) This closes: [#51140](https://github.com/ClickHouse/ClickHouse/issues/51140) This closes: [#47517](https://github.com/ClickHouse/ClickHouse/issues/47517) This closes: [#47189](https://github.com/ClickHouse/ClickHouse/issues/47189) This closes: [#46598](https://github.com/ClickHouse/ClickHouse/issues/46598) This closes: [#40257](https://github.com/ClickHouse/ClickHouse/issues/40257) This closes: [#36504](https://github.com/ClickHouse/ClickHouse/issues/36504) This closes: [#35485](https://github.com/ClickHouse/ClickHouse/issues/35485) This closes: [#33702](https://github.com/ClickHouse/ClickHouse/issues/33702) This closes: [#26702](https://github.com/ClickHouse/ClickHouse/issues/26702) ### Documentation entry for user-facing changes. [#61058](https://github.com/ClickHouse/ClickHouse/pull/61058) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* To increase compatibility with MySQL, function `locate` now accepts arguments `(needle, haystack[, start_pos])` by default. The previous behavior `(haystack, needle, [, start_pos])` can be restored by setting `function_locate_has_mysql_compatible_argument_order = 0`. [#61092](https://github.com/ClickHouse/ClickHouse/pull/61092) ([Robert Schulze](https://github.com/rschu1ze)). +* The obsolete in-memory data parts have been deprecated since version 23.5 and have not been supported since version 23.10. Now the remaining code is removed. Continuation of [#55186](https://github.com/ClickHouse/ClickHouse/issues/55186) and [#45409](https://github.com/ClickHouse/ClickHouse/issues/45409). It is unlikely that you have used in-memory data parts because they were available only before version 23.5 and only when you enabled them manually by specifying the corresponding SETTINGS for a MergeTree table. To check if you have in-memory data parts, run the following query: `SELECT part_type, count() FROM system.parts GROUP BY part_type ORDER BY part_type`. To disable the usage of in-memory data parts, do `ALTER TABLE ... MODIFY SETTING min_bytes_for_compact_part = DEFAULT, min_rows_for_compact_part = DEFAULT`. Before upgrading from old ClickHouse releases, first check that you don't have in-memory data parts. If there are in-memory data parts, disable them first, then wait while there are no in-memory data parts and continue the upgrade. [#61127](https://github.com/ClickHouse/ClickHouse/pull/61127) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Forbid `SimpleAggregateFunction` in `ORDER BY` of `MergeTree` tables (like `AggregateFunction` is forbidden, but they are forbidden because they are not comparable) by default (use `allow_suspicious_primary_key` to allow them). [#61399](https://github.com/ClickHouse/ClickHouse/pull/61399) ([Azat Khuzhin](https://github.com/azat)). +* ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. This is controlled by the settings, `output_format_parquet_string_as_string`, `output_format_orc_string_as_string`, `output_format_arrow_string_as_string`. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases. Parquet/ORC/Arrow supports many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools lack support for the faster `lz4` compression method, that's why we set `zstd` by default. This is controlled by the settings `output_format_parquet_compression_method`, `output_format_orc_compression_method`, and `output_format_arrow_compression_method`. We changed the default to `zstd` for Parquet and ORC, but not Arrow (it is emphasized for low-level usages). [#61817](https://github.com/ClickHouse/ClickHouse/pull/61817) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In the new ClickHouse version, the functions `geoDistance`, `greatCircleDistance`, and `greatCircleAngle` will use 64-bit double precision floating point data type for internal calculations and return type if all the arguments are Float64. This closes [#58476](https://github.com/ClickHouse/ClickHouse/issues/58476). In previous versions, the function always used Float32. You can switch to the old behavior by setting `geo_distance_returns_float64_on_float64_arguments` to `false` or setting `compatibility` to `24.2` or earlier. [#61848](https://github.com/ClickHouse/ClickHouse/pull/61848) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Topk/topkweighed support mode, which return count of values and it's error. [#54508](https://github.com/ClickHouse/ClickHouse/pull/54508) ([UnamedRus](https://github.com/UnamedRus)). +* Add generate_series as a table function. This function generates table with an arithmetic progression with natural numbers. [#59390](https://github.com/ClickHouse/ClickHouse/pull/59390) ([divanik](https://github.com/divanik)). +* Support reading and writing backups as tar archives. [#59535](https://github.com/ClickHouse/ClickHouse/pull/59535) ([josh-hildred](https://github.com/josh-hildred)). +* Implemented support for S3Express buckets. [#59965](https://github.com/ClickHouse/ClickHouse/pull/59965) ([Nikita Taranov](https://github.com/nickitat)). +* Allow to attach parts from a different disk * attach partition from the table on other disks using copy instead of hard link (such as instant table) * attach partition using copy when the hard link fails even on the same disk. [#60112](https://github.com/ClickHouse/ClickHouse/pull/60112) ([Unalian](https://github.com/Unalian)). +* Added function `toMillisecond` which returns the millisecond component for values of type`DateTime` or `DateTime64`. [#60281](https://github.com/ClickHouse/ClickHouse/pull/60281) ([Shaun Struwig](https://github.com/Blargian)). +* Make all format names case insensitive, like Tsv, or TSV, or tsv, or even rowbinary. [#60420](https://github.com/ClickHouse/ClickHouse/pull/60420) ([豪肥肥](https://github.com/HowePa)). +* Add four properties to the `StorageMemory` (memory-engine) `min_bytes_to_keep, max_bytes_to_keep, min_rows_to_keep` and `max_rows_to_keep` - Add tests to reflect new changes - Update `memory.md` documentation - Add table `context` property to `MemorySink` to enable access to table parameter bounds. [#60612](https://github.com/ClickHouse/ClickHouse/pull/60612) ([Jake Bamrah](https://github.com/JakeBamrah)). +* Added function `toMillisecond` which returns the millisecond component for values of type`DateTime` or `DateTime64`. [#60649](https://github.com/ClickHouse/ClickHouse/pull/60649) ([Robert Schulze](https://github.com/rschu1ze)). +* Separate limits on number of waiting and executing queries. Added new server setting `max_waiting_queries` that limits the number of queries waiting due to `async_load_databases`. Existing limits on number of executing queries no longer count waiting queries. [#61053](https://github.com/ClickHouse/ClickHouse/pull/61053) ([Sergei Trifonov](https://github.com/serxa)). +* Add support for `ATTACH PARTITION ALL`. [#61107](https://github.com/ClickHouse/ClickHouse/pull/61107) ([Kirill Nikiforov](https://github.com/allmazz)). +* Add a new function, `getClientHTTPHeader`. This closes [#54665](https://github.com/ClickHouse/ClickHouse/issues/54665). Co-authored with @lingtaolf. [#61820](https://github.com/ClickHouse/ClickHouse/pull/61820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Performance Improvement +* Improve the performance of serialized aggregation method when involving multiple [nullable] columns. This is a general version of [#51399](https://github.com/ClickHouse/ClickHouse/issues/51399) that doesn't compromise on abstraction integrity. [#55809](https://github.com/ClickHouse/ClickHouse/pull/55809) ([Amos Bird](https://github.com/amosbird)). +* Lazy build join output to improve performance of ALL join. [#58278](https://github.com/ClickHouse/ClickHouse/pull/58278) ([LiuNeng](https://github.com/liuneng1994)). +* Improvements to aggregate functions ArgMin / ArgMax / any / anyLast / anyHeavy, as well as `ORDER BY {u8/u16/u32/u64/i8/i16/u32/i64) LIMIT 1` queries. [#58640](https://github.com/ClickHouse/ClickHouse/pull/58640) ([Raúl Marín](https://github.com/Algunenano)). +* Trivial optimize on column filter. Avoid those filter columns whoes underlying data type is not number being filtered with `result_size_hint = -1`. Peak memory can be reduced to 44% of the original in some cases. [#59698](https://github.com/ClickHouse/ClickHouse/pull/59698) ([李扬](https://github.com/taiyang-li)). +* If the table's primary key contains mostly useless columns, don't keep them in memory. This is controlled by a new setting `primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns` with the value `0.9` by default, which means: for a composite primary key, if a column changes its value for at least 0.9 of all the times, the next columns after it will be not loaded. [#60255](https://github.com/ClickHouse/ClickHouse/pull/60255) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Execute multiIf function columnarly when result_type's underlying type is number. [#60384](https://github.com/ClickHouse/ClickHouse/pull/60384) ([李扬](https://github.com/taiyang-li)). +* Faster (almost 2x) mutexes (was slower due to ThreadFuzzer). [#60823](https://github.com/ClickHouse/ClickHouse/pull/60823) ([Azat Khuzhin](https://github.com/azat)). +* Move connection drain from prepare to work, and drain multiple connections in parallel. [#60845](https://github.com/ClickHouse/ClickHouse/pull/60845) ([lizhuoyu5](https://github.com/lzydmxy)). +* Optimize insertManyFrom of nullable number or nullable string. [#60846](https://github.com/ClickHouse/ClickHouse/pull/60846) ([李扬](https://github.com/taiyang-li)). +* Optimized function `dotProduct` to omit unnecessary and expensive memory copies. [#60928](https://github.com/ClickHouse/ClickHouse/pull/60928) ([Robert Schulze](https://github.com/rschu1ze)). +* Operations with the filesystem cache will suffer less from the lock contention. [#61066](https://github.com/ClickHouse/ClickHouse/pull/61066) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Optimize ColumnString::replicate and prevent memcpySmallAllowReadWriteOverflow15Impl from being optimized to built-in memcpy. Close [#61074](https://github.com/ClickHouse/ClickHouse/issues/61074). ColumnString::replicate speeds up by 2.46x on x86-64. [#61075](https://github.com/ClickHouse/ClickHouse/pull/61075) ([李扬](https://github.com/taiyang-li)). +* 30x faster printing for 256-bit integers. [#61100](https://github.com/ClickHouse/ClickHouse/pull/61100) ([Raúl Marín](https://github.com/Algunenano)). +* If a query with a syntax error contained COLUMNS matcher with a regular expression, the regular expression was compiled each time during the parser's backtracking, instead of being compiled once. This was a fundamental error. The compiled regexp was put to AST. But the letter A in AST means "abstract" which means it should not contain heavyweight objects. Parts of AST can be created and discarded during parsing, including a large number of backtracking. This leads to slowness on the parsing side and consequently allows DoS by a readonly user. But the main problem is that it prevents progress in fuzzers. [#61543](https://github.com/ClickHouse/ClickHouse/pull/61543) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a new analyzer pass to optimize in single value. [#61564](https://github.com/ClickHouse/ClickHouse/pull/61564) ([LiuNeng](https://github.com/liuneng1994)). + +#### Improvement +* While running the MODIFY COLUMN query for materialized views, check the inner table's structure to ensure every column exists. [#47427](https://github.com/ClickHouse/ClickHouse/pull/47427) ([sunny](https://github.com/sunny19930321)). +* Added table `system.keywords` which contains all the keywords from parser. Mostly needed and will be used for better fuzzing and syntax highlighting. [#51808](https://github.com/ClickHouse/ClickHouse/pull/51808) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Ordinary database engine is deprecated. You will receive a warning in clickhouse-client if your server is using it. This closes [#52229](https://github.com/ClickHouse/ClickHouse/issues/52229). [#56942](https://github.com/ClickHouse/ClickHouse/pull/56942) ([shabroo](https://github.com/shabroo)). +* All zero copy locks related to a table have to be dropped when the table is dropped. The directory which contains these locks has to be removed also. [#57575](https://github.com/ClickHouse/ClickHouse/pull/57575) ([Sema Checherinda](https://github.com/CheSema)). +* Allow declaring enum in external table structure. [#57857](https://github.com/ClickHouse/ClickHouse/pull/57857) ([Duc Canh Le](https://github.com/canhld94)). +* Consider lightweight deleted rows when selecting parts to merge. [#58223](https://github.com/ClickHouse/ClickHouse/pull/58223) ([Zhuo Qiu](https://github.com/jewelzqiu)). +* This PR makes http/https connections reusable for all uses cases. Even when response is 3xx or 4xx. [#58845](https://github.com/ClickHouse/ClickHouse/pull/58845) ([Sema Checherinda](https://github.com/CheSema)). +* Added comments for columns for more system tables. Continuation of https://github.com/ClickHouse/ClickHouse/pull/58356. [#59016](https://github.com/ClickHouse/ClickHouse/pull/59016) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Now we can use virtual columns in PREWHERE. It's worthwhile for non-const virtual columns like `_part_offset`. [#59033](https://github.com/ClickHouse/ClickHouse/pull/59033) ([Amos Bird](https://github.com/amosbird)). +* Add ability to skip read-only replicas for INSERT into Distributed engine (Controlled with `distributed_insert_skip_read_only_replicas` setting, by default OFF - backward compatible). [#59176](https://github.com/ClickHouse/ClickHouse/pull/59176) ([Azat Khuzhin](https://github.com/azat)). +* Instead using a constant key, now object storage generates key for determining remove objects capability. [#59495](https://github.com/ClickHouse/ClickHouse/pull/59495) ([Sema Checherinda](https://github.com/CheSema)). +* Add positional pread in libhdfs3. If you want to call positional read in libhdfs3, use the hdfsPread function in hdfs.h as follows. `tSize hdfsPread(hdfsFS fs, hdfsFile file, void * buffer, tSize length, tOffset position);`. [#59624](https://github.com/ClickHouse/ClickHouse/pull/59624) ([M1eyu](https://github.com/M1eyu2018)). +* Add asynchronous WriteBuffer for AzureBlobStorage similar to S3. [#59929](https://github.com/ClickHouse/ClickHouse/pull/59929) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Allow "local" as object storage type instead of "local_blob_storage". [#60165](https://github.com/ClickHouse/ClickHouse/pull/60165) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improved overall usability of virtual columns. Now it is allowed to use virtual columns in `PREWHERE` (it's worthwhile for non-const virtual columns like `_part_offset`). Now a builtin documentation is available for virtual columns as a comment of column in `DESCRIBE` query with enabled setting `describe_include_virtual_columns`. [#60205](https://github.com/ClickHouse/ClickHouse/pull/60205) ([Anton Popov](https://github.com/CurtizJ)). +* Parallel flush of pending INSERT blocks of Distributed engine on `DETACH`/server shutdown and `SYSTEM FLUSH DISTRIBUTED` (Parallelism will work only if you have multi disk policy for table (like everything in Distributed engine right now)). [#60225](https://github.com/ClickHouse/ClickHouse/pull/60225) ([Azat Khuzhin](https://github.com/azat)). +* Filter setting is improper in `joinRightColumnsSwitchNullability`, resolve [#59625](https://github.com/ClickHouse/ClickHouse/issues/59625). [#60259](https://github.com/ClickHouse/ClickHouse/pull/60259) ([lgbo](https://github.com/lgbo-ustc)). +* Add a setting to force read-through cache for merges. [#60308](https://github.com/ClickHouse/ClickHouse/pull/60308) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Issue [#57598](https://github.com/ClickHouse/ClickHouse/issues/57598) mentions a variant behaviour regarding transaction handling. An issued COMMIT/ROLLBACK when no transaction is active is reported as an error contrary to MySQL behaviour. [#60338](https://github.com/ClickHouse/ClickHouse/pull/60338) ([PapaToemmsn](https://github.com/PapaToemmsn)). +* Added `none_only_active` mode for `distributed_ddl_output_mode` setting. [#60340](https://github.com/ClickHouse/ClickHouse/pull/60340) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Allow configuring HTTP redirect handlers for clickhouse-server. For example, you can make `/` redirect to the Play UI. [#60390](https://github.com/ClickHouse/ClickHouse/pull/60390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The advanced dashboard has slightly better colors for multi-line graphs. [#60391](https://github.com/ClickHouse/ClickHouse/pull/60391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Function `substring` now has a new alias `byteSlice`. [#60494](https://github.com/ClickHouse/ClickHouse/pull/60494) ([Robert Schulze](https://github.com/rschu1ze)). +* Renamed server setting `dns_cache_max_size` to `dns_cache_max_entries` to reduce ambiguity. [#60500](https://github.com/ClickHouse/ClickHouse/pull/60500) ([Kirill Nikiforov](https://github.com/allmazz)). +* `SHOW INDEX | INDEXES | INDICES | KEYS` no longer sorts by the primary key columns (which was unintuitive). [#60514](https://github.com/ClickHouse/ClickHouse/pull/60514) ([Robert Schulze](https://github.com/rschu1ze)). +* Keeper improvement: abort during startup if an invalid snapshot is detected to avoid data loss. [#60537](https://github.com/ClickHouse/ClickHouse/pull/60537) ([Antonio Andelic](https://github.com/antonio2368)). +* Added MergeTree read split ranges into intersecting and non intersecting fault injection using `merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_fault_probability` setting. [#60548](https://github.com/ClickHouse/ClickHouse/pull/60548) ([Maksim Kita](https://github.com/kitaisreal)). +* The Advanced dashboard now has controls always visible on scrolling. This allows you to add a new chart without scrolling up. [#60692](https://github.com/ClickHouse/ClickHouse/pull/60692) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* String types and Enums can be used in the same context, such as: arrays, UNION queries, conditional expressions. This closes [#60726](https://github.com/ClickHouse/ClickHouse/issues/60726). [#60727](https://github.com/ClickHouse/ClickHouse/pull/60727) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update tzdata to 2024a. [#60768](https://github.com/ClickHouse/ClickHouse/pull/60768) ([Raúl Marín](https://github.com/Algunenano)). +* Support files without format extension in Filesystem database. [#60795](https://github.com/ClickHouse/ClickHouse/pull/60795) ([Kruglov Pavel](https://github.com/Avogar)). +* Keeper improvement: support `leadership_expiry_ms` in Keeper's settings. [#60806](https://github.com/ClickHouse/ClickHouse/pull/60806) ([Brokenice0415](https://github.com/Brokenice0415)). +* Always infer exponential numbers in JSON formats regardless of the setting `input_format_try_infer_exponent_floats`. Add setting `input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects` that allows to use String type for ambiguous paths instead of an exception during named Tuples inference from JSON objects. [#60808](https://github.com/ClickHouse/ClickHouse/pull/60808) ([Kruglov Pavel](https://github.com/Avogar)). +* Add support for `START TRANSACTION` syntax typically used in MySQL syntax, resolving https://github.com/ClickHouse/ClickHouse/discussions/60865. [#60886](https://github.com/ClickHouse/ClickHouse/pull/60886) ([Zach Naimon](https://github.com/ArctypeZach)). +* Add a flag for SMJ to treat null as biggest/smallest. So the behavior can be compitable with other SQL systems, like Apache Spark. [#60896](https://github.com/ClickHouse/ClickHouse/pull/60896) ([loudongfeng](https://github.com/loudongfeng)). +* Clickhouse version has been added to docker labels. Closes [#54224](https://github.com/ClickHouse/ClickHouse/issues/54224). [#60949](https://github.com/ClickHouse/ClickHouse/pull/60949) ([Nikolay Monkov](https://github.com/nikmonkov)). +* Add a setting `parallel_replicas_allow_in_with_subquery = 1` which allows subqueries for IN work with parallel replicas. [#60950](https://github.com/ClickHouse/ClickHouse/pull/60950) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* DNSResolver shuffles set of resolved IPs. [#60965](https://github.com/ClickHouse/ClickHouse/pull/60965) ([Sema Checherinda](https://github.com/CheSema)). +* Support detect output format by file exctension in `clickhouse-client` and `clickhouse-local`. [#61036](https://github.com/ClickHouse/ClickHouse/pull/61036) ([豪肥肥](https://github.com/HowePa)). +* Check memory limit update periodically. [#61049](https://github.com/ClickHouse/ClickHouse/pull/61049) ([Han Fei](https://github.com/hanfei1991)). +* Enable processors profiling (time spent/in and out bytes for sorting, aggregation, ...) by default. [#61096](https://github.com/ClickHouse/ClickHouse/pull/61096) ([Azat Khuzhin](https://github.com/azat)). +* Add the function `toUInt128OrZero`, which was missed by mistake (the mistake is related to https://github.com/ClickHouse/ClickHouse/pull/945). The compatibility aliases `FROM_UNIXTIME` and `DATE_FORMAT` (they are not ClickHouse-native and only exist for MySQL compatibility) have been made case insensitive, as expected for SQL-compatibility aliases. [#61114](https://github.com/ClickHouse/ClickHouse/pull/61114) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improvements for the access checks, allowing to revoke of unpossessed rights in case the target user doesn't have the revoking grants either. Example: ```sql GRANT SELECT ON *.* TO user1; REVOKE SELECT ON system.* FROM user1;. [#61115](https://github.com/ClickHouse/ClickHouse/pull/61115) ([pufit](https://github.com/pufit)). +* Fix an error in previeous opt: https://github.com/ClickHouse/ClickHouse/pull/59698: remove break to make sure the first filtered column has minimum size cc @jsc0218. [#61145](https://github.com/ClickHouse/ClickHouse/pull/61145) ([李扬](https://github.com/taiyang-li)). +* Fix `has()` function with `Nullable` column (fixes [#60214](https://github.com/ClickHouse/ClickHouse/issues/60214)). [#61249](https://github.com/ClickHouse/ClickHouse/pull/61249) ([Mikhail Koviazin](https://github.com/mkmkme)). +* Now it's possible to specify attribute `merge="true"` in config substitutions for subtrees ``. In case this attribute specified, clickhouse will merge subtree with existing configuration, otherwise default behavior is append new content to configuration. [#61299](https://github.com/ClickHouse/ClickHouse/pull/61299) ([alesapin](https://github.com/alesapin)). +* Add async metrics for virtual memory mappings: VMMaxMapCount & VMNumMaps. Closes [#60662](https://github.com/ClickHouse/ClickHouse/issues/60662). [#61354](https://github.com/ClickHouse/ClickHouse/pull/61354) ([Tuan Pham Anh](https://github.com/tuanpavn)). +* Use `temporary_files_codec` setting in all places where we create temporary data, for example external memory sorting and external memory GROUP BY. Before it worked only in `partial_merge` JOIN algorithm. [#61456](https://github.com/ClickHouse/ClickHouse/pull/61456) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove duplicated check `containing_part.empty()`, It's already being checked here: https://github.com/ClickHouse/ClickHouse/blob/1296dac3c7e47670872c15e3f5e58f869e0bd2f2/src/Storages/MergeTree/MergeTreeData.cpp#L6141. [#61467](https://github.com/ClickHouse/ClickHouse/pull/61467) ([William Schoeffel](https://github.com/wiledusc)). +* Add a new setting `max_parser_backtracks` which allows to limit the complexity of query parsing. [#61502](https://github.com/ClickHouse/ClickHouse/pull/61502) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support parallel reading for azure blob storage. [#61503](https://github.com/ClickHouse/ClickHouse/pull/61503) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Less contention during dynamic resize of filesystem cache. [#61524](https://github.com/ClickHouse/ClickHouse/pull/61524) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disallow sharded mode of StorageS3 queue, because it will be rewritten. [#61537](https://github.com/ClickHouse/ClickHouse/pull/61537) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed typo: from `use_leagcy_max_level` to `use_legacy_max_level`. [#61545](https://github.com/ClickHouse/ClickHouse/pull/61545) ([William Schoeffel](https://github.com/wiledusc)). +* Remove some duplicate entries in blob_storage_log. [#61622](https://github.com/ClickHouse/ClickHouse/pull/61622) ([YenchangChan](https://github.com/YenchangChan)). +* Enable `allow_experimental_analyzer` setting by default. [#61652](https://github.com/ClickHouse/ClickHouse/pull/61652) ([Dmitry Novik](https://github.com/novikd)). +* Added `current_user` function as a compatibility alias for MySQL. [#61770](https://github.com/ClickHouse/ClickHouse/pull/61770) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Use managed identity for backups IO when using Azure Blob Storage. Add a setting to prevent ClickHouse from attempting to create a non-existent container, which requires permissions at the storage account level. [#61785](https://github.com/ClickHouse/ClickHouse/pull/61785) ([Daniel Pozo Escalona](https://github.com/danipozo)). +* Enable `output_format_pretty_row_numbers` by default. It is better for usability. [#61791](https://github.com/ClickHouse/ClickHouse/pull/61791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In the previous version, some numbers in Pretty formats were not pretty enough. [#61794](https://github.com/ClickHouse/ClickHouse/pull/61794) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A long value in Pretty formats won't be cut if it is the single value in the resultset, such as in the result of the `SHOW CREATE TABLE` query. [#61795](https://github.com/ClickHouse/ClickHouse/pull/61795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Similarly to `clickhouse-local`, `clickhouse-client` will accept the `--output-format` option as a synonym to the `--format` option. This closes [#59848](https://github.com/ClickHouse/ClickHouse/issues/59848). [#61797](https://github.com/ClickHouse/ClickHouse/pull/61797) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* If stdout is a terminal and the output format is not specified, `clickhouse-client` and similar tools will use `PrettyCompact` by default, similarly to the interactive mode. `clickhouse-client` and `clickhouse-local` will handle command line arguments for input and output formats in a unified fashion. This closes [#61272](https://github.com/ClickHouse/ClickHouse/issues/61272). [#61800](https://github.com/ClickHouse/ClickHouse/pull/61800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Underscore digit groups in Pretty formats for better readability. This is controlled by a new setting, `output_format_pretty_highlight_digit_groups`. [#61802](https://github.com/ClickHouse/ClickHouse/pull/61802) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add ability to override initial INSERT SETTINGS via SYSTEM FLUSH DISTRIBUTED. [#61832](https://github.com/ClickHouse/ClickHouse/pull/61832) ([Azat Khuzhin](https://github.com/azat)). +* Fixed grammar from "a" to "the" in the warning message. There is only one Atomic engine, so it should be "to the new Atomic engine" instead of "to a new Atomic engine". [#61952](https://github.com/ClickHouse/ClickHouse/pull/61952) ([shabroo](https://github.com/shabroo)). + +#### Build/Testing/Packaging Improvement +* Update sccache to the latest version; significantly reduce images size by reshaking the dependency trees; use the latest working odbc driver. [#59953](https://github.com/ClickHouse/ClickHouse/pull/59953) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update python related style checkers. Continue the [#50174](https://github.com/ClickHouse/ClickHouse/issues/50174). [#60408](https://github.com/ClickHouse/ClickHouse/pull/60408) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Upgrade `prqlc` to 0.11.3. [#60616](https://github.com/ClickHouse/ClickHouse/pull/60616) ([Maximilian Roos](https://github.com/max-sixty)). +* Attach gdb to running fuzzer process. [#60654](https://github.com/ClickHouse/ClickHouse/pull/60654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use explicit template instantiation more aggressively. Get rid of templates in favor of overloaded functions in some places. [#60730](https://github.com/ClickHouse/ClickHouse/pull/60730) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* The real-time query profiler now works on AArch64. In previous versions, it worked only when a program didn't spend time inside a syscall. [#60807](https://github.com/ClickHouse/ClickHouse/pull/60807) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ... Too big translation unit in `Aggregator`. [#61211](https://github.com/ClickHouse/ClickHouse/pull/61211) ([lgbo](https://github.com/lgbo-ustc)). +* Fixed flakiness of 01603_insert_select_too_many_parts test. Closes [#61158](https://github.com/ClickHouse/ClickHouse/issues/61158). [#61259](https://github.com/ClickHouse/ClickHouse/pull/61259) ([Ilya Yatsishin](https://github.com/qoega)). +* Now it possible to use `chassert(expression, comment)` in the codebase. [#61263](https://github.com/ClickHouse/ClickHouse/pull/61263) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Teach the fuzzer to use other numeric types. [#61317](https://github.com/ClickHouse/ClickHouse/pull/61317) ([Raúl Marín](https://github.com/Algunenano)). +* Increase memory limit for coverage builds. [#61405](https://github.com/ClickHouse/ClickHouse/pull/61405) ([Raúl Marín](https://github.com/Algunenano)). +* Add generic query text fuzzer in `clickhouse-local`. [#61508](https://github.com/ClickHouse/ClickHouse/pull/61508) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix function execution over const and LowCardinality with GROUP BY const for analyzer [#59986](https://github.com/ClickHouse/ClickHouse/pull/59986) ([Azat Khuzhin](https://github.com/azat)). +* Fix finished_mutations_to_keep=0 for MergeTree (as docs says 0 is to keep everything) [#60031](https://github.com/ClickHouse/ClickHouse/pull/60031) ([Azat Khuzhin](https://github.com/azat)). +* PartsSplitter invalid ranges for the same part [#60041](https://github.com/ClickHouse/ClickHouse/pull/60041) ([Maksim Kita](https://github.com/kitaisreal)). +* Azure Blob Storage : Fix issues endpoint and prefix [#60251](https://github.com/ClickHouse/ClickHouse/pull/60251) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* fix LRUResource Cache bug (Hive cache) [#60262](https://github.com/ClickHouse/ClickHouse/pull/60262) ([shanfengp](https://github.com/Aed-p)). +* Force reanalysis if parallel replicas changed [#60362](https://github.com/ClickHouse/ClickHouse/pull/60362) ([Raúl Marín](https://github.com/Algunenano)). +* Fix usage of plain metadata type with new disks configuration option [#60396](https://github.com/ClickHouse/ClickHouse/pull/60396) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Try to fix logical error 'Cannot capture column because it has incompatible type' in mapContainsKeyLike [#60451](https://github.com/ClickHouse/ClickHouse/pull/60451) ([Kruglov Pavel](https://github.com/Avogar)). +* Try to avoid calculation of scalar subqueries for CREATE TABLE. [#60464](https://github.com/ClickHouse/ClickHouse/pull/60464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix deadlock in parallel parsing when lots of rows are skipped due to errors [#60516](https://github.com/ClickHouse/ClickHouse/pull/60516) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix_max_query_size_for_kql_compound_operator: [#60534](https://github.com/ClickHouse/ClickHouse/pull/60534) ([Yong Wang](https://github.com/kashwy)). +* Keeper fix: add timeouts when waiting for commit logs [#60544](https://github.com/ClickHouse/ClickHouse/pull/60544) ([Antonio Andelic](https://github.com/antonio2368)). +* Reduce the number of read rows from `system.numbers` [#60546](https://github.com/ClickHouse/ClickHouse/pull/60546) ([JackyWoo](https://github.com/JackyWoo)). +* Don't output number tips for date types [#60577](https://github.com/ClickHouse/ClickHouse/pull/60577) ([Raúl Marín](https://github.com/Algunenano)). +* Fix reading from MergeTree with non-deterministic functions in filter [#60586](https://github.com/ClickHouse/ClickHouse/pull/60586) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix logical error on bad compatibility setting value type [#60596](https://github.com/ClickHouse/ClickHouse/pull/60596) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix inconsistent aggregate function states in mixed x86-64 / ARM clusters [#60610](https://github.com/ClickHouse/ClickHouse/pull/60610) ([Harry Lee](https://github.com/HarryLeeIBM)). +* fix(prql): Robust panic handler [#60615](https://github.com/ClickHouse/ClickHouse/pull/60615) ([Maximilian Roos](https://github.com/max-sixty)). +* Fix `intDiv` for decimal and date arguments [#60672](https://github.com/ClickHouse/ClickHouse/pull/60672) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Fix: expand CTE in alter modify query [#60682](https://github.com/ClickHouse/ClickHouse/pull/60682) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix system.parts for non-Atomic/Ordinary database engine (i.e. Memory) [#60689](https://github.com/ClickHouse/ClickHouse/pull/60689) ([Azat Khuzhin](https://github.com/azat)). +* Fix "Invalid storage definition in metadata file" for parameterized views [#60708](https://github.com/ClickHouse/ClickHouse/pull/60708) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer overflow in CompressionCodecMultiple [#60731](https://github.com/ClickHouse/ClickHouse/pull/60731) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove nonsense from SQL/JSON [#60738](https://github.com/ClickHouse/ClickHouse/pull/60738) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove wrong sanitize checking in aggregate function quantileGK [#60740](https://github.com/ClickHouse/ClickHouse/pull/60740) ([李扬](https://github.com/taiyang-li)). +* Fix insert-select + insert_deduplication_token bug by setting streams to 1 [#60745](https://github.com/ClickHouse/ClickHouse/pull/60745) ([Jordi Villar](https://github.com/jrdi)). +* Prevent setting custom metadata headers on unsupported multipart upload operations [#60748](https://github.com/ClickHouse/ClickHouse/pull/60748) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)). +* Fix toStartOfInterval [#60763](https://github.com/ClickHouse/ClickHouse/pull/60763) ([Andrey Zvonov](https://github.com/zvonand)). +* Fix crash in arrayEnumerateRanked [#60764](https://github.com/ClickHouse/ClickHouse/pull/60764) ([Raúl Marín](https://github.com/Algunenano)). +* Fix crash when using input() in INSERT SELECT JOIN [#60765](https://github.com/ClickHouse/ClickHouse/pull/60765) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash with different allow_experimental_analyzer value in subqueries [#60770](https://github.com/ClickHouse/ClickHouse/pull/60770) ([Dmitry Novik](https://github.com/novikd)). +* Remove recursion when reading from S3 [#60849](https://github.com/ClickHouse/ClickHouse/pull/60849) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix possible stuck on error in HashedDictionaryParallelLoader [#60926](https://github.com/ClickHouse/ClickHouse/pull/60926) ([vdimir](https://github.com/vdimir)). +* Fix async RESTORE with Replicated database [#60934](https://github.com/ClickHouse/ClickHouse/pull/60934) ([Antonio Andelic](https://github.com/antonio2368)). +* fix csv format not support tuple [#60994](https://github.com/ClickHouse/ClickHouse/pull/60994) ([shuai.xu](https://github.com/shuai-xu)). +* Fix deadlock in async inserts to `Log` tables via native protocol [#61055](https://github.com/ClickHouse/ClickHouse/pull/61055) ([Anton Popov](https://github.com/CurtizJ)). +* Fix lazy execution of default argument in dictGetOrDefault for RangeHashedDictionary [#61196](https://github.com/ClickHouse/ClickHouse/pull/61196) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix multiple bugs in groupArraySorted [#61203](https://github.com/ClickHouse/ClickHouse/pull/61203) ([Raúl Marín](https://github.com/Algunenano)). +* Fix Keeper reconfig for standalone binary [#61233](https://github.com/ClickHouse/ClickHouse/pull/61233) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix usage of session_token in S3 engine [#61234](https://github.com/ClickHouse/ClickHouse/pull/61234) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix possible incorrect result of aggregate function `uniqExact` [#61257](https://github.com/ClickHouse/ClickHouse/pull/61257) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bugs in show database [#61269](https://github.com/ClickHouse/ClickHouse/pull/61269) ([Raúl Marín](https://github.com/Algunenano)). +* Fix logical error in RabbitMQ storage with MATERIALIZED columns [#61320](https://github.com/ClickHouse/ClickHouse/pull/61320) ([vdimir](https://github.com/vdimir)). +* Fix CREATE OR REPLACE DICTIONARY [#61356](https://github.com/ClickHouse/ClickHouse/pull/61356) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix crash in ObjectJson parsing array with nulls [#61364](https://github.com/ClickHouse/ClickHouse/pull/61364) ([vdimir](https://github.com/vdimir)). +* Fix ATTACH query with external ON CLUSTER [#61365](https://github.com/ClickHouse/ClickHouse/pull/61365) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix consecutive keys optimization for nullable keys [#61393](https://github.com/ClickHouse/ClickHouse/pull/61393) ([Anton Popov](https://github.com/CurtizJ)). +* fix issue of actions dag split [#61458](https://github.com/ClickHouse/ClickHouse/pull/61458) ([Raúl Marín](https://github.com/Algunenano)). +* Fix finishing a failed RESTORE [#61466](https://github.com/ClickHouse/ClickHouse/pull/61466) ([Vitaly Baranov](https://github.com/vitlibar)). +* Disable async_insert_use_adaptive_busy_timeout correctly with compatibility settings [#61468](https://github.com/ClickHouse/ClickHouse/pull/61468) ([Raúl Marín](https://github.com/Algunenano)). +* Allow queuing in restore pool [#61475](https://github.com/ClickHouse/ClickHouse/pull/61475) ([Nikita Taranov](https://github.com/nickitat)). +* Fix bug when reading system.parts using UUID (issue 61220). [#61479](https://github.com/ClickHouse/ClickHouse/pull/61479) ([Dan Wu](https://github.com/wudanzy)). +* Fix ALTER QUERY MODIFY SQL SECURITY [#61480](https://github.com/ClickHouse/ClickHouse/pull/61480) ([pufit](https://github.com/pufit)). +* Fix crash in window view [#61526](https://github.com/ClickHouse/ClickHouse/pull/61526) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `repeat` with non native integers [#61527](https://github.com/ClickHouse/ClickHouse/pull/61527) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix client `-s` argument [#61530](https://github.com/ClickHouse/ClickHouse/pull/61530) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Reset part level upon attach from disk on MergeTree [#61536](https://github.com/ClickHouse/ClickHouse/pull/61536) ([Arthur Passos](https://github.com/arthurpassos)). +* Fix crash in arrayPartialReverseSort [#61539](https://github.com/ClickHouse/ClickHouse/pull/61539) ([Raúl Marín](https://github.com/Algunenano)). +* Fix string search with const position [#61547](https://github.com/ClickHouse/ClickHouse/pull/61547) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix addDays cause an error when used datetime64 [#61561](https://github.com/ClickHouse/ClickHouse/pull/61561) ([Shuai li](https://github.com/loneylee)). +* disallow LowCardinality input type for JSONExtract [#61617](https://github.com/ClickHouse/ClickHouse/pull/61617) ([Julia Kartseva](https://github.com/jkartseva)). +* Fix `system.part_log` for async insert with deduplication [#61620](https://github.com/ClickHouse/ClickHouse/pull/61620) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix Non-ready set for system.parts. [#61666](https://github.com/ClickHouse/ClickHouse/pull/61666) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Don't allow the same expression in ORDER BY with and without WITH FILL [#61667](https://github.com/ClickHouse/ClickHouse/pull/61667) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix actual_part_name for REPLACE_RANGE (`Entry actual part isn't empty yet`) [#61675](https://github.com/ClickHouse/ClickHouse/pull/61675) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix columns after executing MODIFY QUERY for a materialized view with internal table [#61734](https://github.com/ClickHouse/ClickHouse/pull/61734) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix crash in `multiSearchAllPositionsCaseInsensitiveUTF8` for incorrect UTF-8 [#61749](https://github.com/ClickHouse/ClickHouse/pull/61749) ([pufit](https://github.com/pufit)). +* Fix RANGE frame is not supported for Nullable columns. [#61766](https://github.com/ClickHouse/ClickHouse/pull/61766) ([YuanLiu](https://github.com/ditgittube)). +* Revert "Revert "Fix bug when reading system.parts using UUID (issue 61220)."" [#61779](https://github.com/ClickHouse/ClickHouse/pull/61779) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). + +#### CI Fix or Improvement (changelog entry is not required) + +* Decoupled changes from [#60408](https://github.com/ClickHouse/ClickHouse/issues/60408). [#60553](https://github.com/ClickHouse/ClickHouse/pull/60553) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Eliminates the need to provide input args to docker server jobs to clean yml files. [#60602](https://github.com/ClickHouse/ClickHouse/pull/60602) ([Max K.](https://github.com/maxknv)). +* Debug and fix markreleaseready. [#60611](https://github.com/ClickHouse/ClickHouse/pull/60611) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix build_report job so that it's defined by ci_config only (not yml file). [#60613](https://github.com/ClickHouse/ClickHouse/pull/60613) ([Max K.](https://github.com/maxknv)). +* Do not await ci pending jobs on release branches decrease wait timeout to fit into gh job timeout. [#60652](https://github.com/ClickHouse/ClickHouse/pull/60652) ([Max K.](https://github.com/maxknv)). +* Set limited number of builds for "special build check" report in backports. [#60850](https://github.com/ClickHouse/ClickHouse/pull/60850) ([Max K.](https://github.com/maxknv)). +* ... [#60935](https://github.com/ClickHouse/ClickHouse/pull/60935) ([Max K.](https://github.com/maxknv)). +* ... [#60947](https://github.com/ClickHouse/ClickHouse/pull/60947) ([Max K.](https://github.com/maxknv)). +* ... [#60952](https://github.com/ClickHouse/ClickHouse/pull/60952) ([Max K.](https://github.com/maxknv)). +* ... [#60958](https://github.com/ClickHouse/ClickHouse/pull/60958) ([Max K.](https://github.com/maxknv)). +* ... [#61022](https://github.com/ClickHouse/ClickHouse/pull/61022) ([Max K.](https://github.com/maxknv)). +* Just a preparation for the merge queue support. [#61099](https://github.com/ClickHouse/ClickHouse/pull/61099) ([Max K.](https://github.com/maxknv)). +* ... [#61133](https://github.com/ClickHouse/ClickHouse/pull/61133) ([Max K.](https://github.com/maxknv)). +* In PRs: - run typos, aspell check - always - run pylint, mypy - only if py file(s) changed in PRs - run basic source files style check - only if not all changes in py files. [#61148](https://github.com/ClickHouse/ClickHouse/pull/61148) ([Max K.](https://github.com/maxknv)). +* ... [#61172](https://github.com/ClickHouse/ClickHouse/pull/61172) ([Max K.](https://github.com/maxknv)). +* ... [#61183](https://github.com/ClickHouse/ClickHouse/pull/61183) ([Han Fei](https://github.com/hanfei1991)). +* ... [#61185](https://github.com/ClickHouse/ClickHouse/pull/61185) ([Max K.](https://github.com/maxknv)). +* TBD. [#61197](https://github.com/ClickHouse/ClickHouse/pull/61197) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* ... [#61214](https://github.com/ClickHouse/ClickHouse/pull/61214) ([Max K.](https://github.com/maxknv)). +* ... [#61441](https://github.com/ClickHouse/ClickHouse/pull/61441) ([Max K.](https://github.com/maxknv)). +* ![Screenshot_20240323_025055](https://github.com/ClickHouse/ClickHouse/assets/18581488/ccaab212-a1d3-4dfb-8d56-b1991760b6bf). [#61801](https://github.com/ClickHouse/ClickHouse/pull/61801) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ... [#61877](https://github.com/ClickHouse/ClickHouse/pull/61877) ([Max K.](https://github.com/maxknv)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Revert "Use `MergeTree` as a default table engine""'. [#60524](https://github.com/ClickHouse/ClickHouse/pull/60524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Revert "Support resource request canceling""'. [#60558](https://github.com/ClickHouse/ClickHouse/pull/60558) ([Sergei Trifonov](https://github.com/serxa)). +* NO CL ENTRY: 'Revert "Add `toMillisecond` function"'. [#60644](https://github.com/ClickHouse/ClickHouse/pull/60644) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Synchronize parsers"'. [#60759](https://github.com/ClickHouse/ClickHouse/pull/60759) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Fix wacky primary key sorting in `SHOW INDEX`"'. [#60898](https://github.com/ClickHouse/ClickHouse/pull/60898) ([Antonio Andelic](https://github.com/antonio2368)). +* NO CL ENTRY: 'Revert "CI: make style check faster"'. [#61142](https://github.com/ClickHouse/ClickHouse/pull/61142) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Don't allow to set max_parallel_replicas to 0 as it doesn't make sense"'. [#61200](https://github.com/ClickHouse/ClickHouse/pull/61200) ([Kruglov Pavel](https://github.com/Avogar)). +* NO CL ENTRY: 'Revert "Fix usage of session_token in S3 engine"'. [#61359](https://github.com/ClickHouse/ClickHouse/pull/61359) ([Antonio Andelic](https://github.com/antonio2368)). +* NO CL ENTRY: 'Revert "Revert "Fix usage of session_token in S3 engine""'. [#61362](https://github.com/ClickHouse/ClickHouse/pull/61362) ([Kruglov Pavel](https://github.com/Avogar)). +* NO CL ENTRY: 'Reorder hidden and shown checks in comment, change url of Mergeable check'. [#61373](https://github.com/ClickHouse/ClickHouse/pull/61373) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* NO CL ENTRY: 'Remove unnecessary layers from clickhouse/cctools'. [#61374](https://github.com/ClickHouse/ClickHouse/pull/61374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* NO CL ENTRY: 'Revert "Updated format settings references in the docs (datetime.md)"'. [#61435](https://github.com/ClickHouse/ClickHouse/pull/61435) ([Kruglov Pavel](https://github.com/Avogar)). +* NO CL ENTRY: 'Revert "CI: ARM integration tests: disable tests with HDFS "'. [#61449](https://github.com/ClickHouse/ClickHouse/pull/61449) ([Max K.](https://github.com/maxknv)). +* NO CL ENTRY: 'Revert "Analyzer: Fix virtual columns in StorageMerge"'. [#61518](https://github.com/ClickHouse/ClickHouse/pull/61518) ([Antonio Andelic](https://github.com/antonio2368)). +* NO CL ENTRY: 'Revert "Revert "Analyzer: Fix virtual columns in StorageMerge""'. [#61528](https://github.com/ClickHouse/ClickHouse/pull/61528) ([Dmitry Novik](https://github.com/novikd)). +* NO CL ENTRY: 'Improve build_download_helper'. [#61592](https://github.com/ClickHouse/ClickHouse/pull/61592) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* NO CL ENTRY: 'Revert "Un-flake `test_undrop_query`"'. [#61668](https://github.com/ClickHouse/ClickHouse/pull/61668) ([Robert Schulze](https://github.com/rschu1ze)). +* NO CL ENTRY: 'Fix flaky tests (stateless, integration)'. [#61816](https://github.com/ClickHouse/ClickHouse/pull/61816) ([Nikita Fomichev](https://github.com/fm4v)). +* NO CL ENTRY: 'Better usability of "expect" tests: less trouble with running directly'. [#61818](https://github.com/ClickHouse/ClickHouse/pull/61818) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Fix flaky `02122_parallel_formatting_Template`"'. [#61868](https://github.com/ClickHouse/ClickHouse/pull/61868) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Add --now option to enable and start the service" #job_Install_packages_amd64'. [#61878](https://github.com/ClickHouse/ClickHouse/pull/61878) ([Max K.](https://github.com/maxknv)). +* NO CL ENTRY: 'Revert "disallow LowCardinality input type for JSONExtract"'. [#61960](https://github.com/ClickHouse/ClickHouse/pull/61960) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve query performance in case of very small blocks [#58879](https://github.com/ClickHouse/ClickHouse/pull/58879) ([Azat Khuzhin](https://github.com/azat)). +* Analyzer: fixes for JOIN columns resolution [#59007](https://github.com/ClickHouse/ClickHouse/pull/59007) ([vdimir](https://github.com/vdimir)). +* Fix race on `Context::async_insert_queue` [#59082](https://github.com/ClickHouse/ClickHouse/pull/59082) ([Alexander Tokmakov](https://github.com/tavplubix)). +* CI: support batch specification in commit message [#59738](https://github.com/ClickHouse/ClickHouse/pull/59738) ([Max K.](https://github.com/maxknv)). +* Update storing-data.md [#60024](https://github.com/ClickHouse/ClickHouse/pull/60024) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make max_insert_delayed_streams_for_parallel_write actually work [#60079](https://github.com/ClickHouse/ClickHouse/pull/60079) ([alesapin](https://github.com/alesapin)). +* Analyzer: support join using column from select list [#60182](https://github.com/ClickHouse/ClickHouse/pull/60182) ([vdimir](https://github.com/vdimir)). +* test for [#60223](https://github.com/ClickHouse/ClickHouse/issues/60223) [#60258](https://github.com/ClickHouse/ClickHouse/pull/60258) ([Denny Crane](https://github.com/den-crane)). +* Analyzer: Refactor execution name for ConstantNode [#60313](https://github.com/ClickHouse/ClickHouse/pull/60313) ([Dmitry Novik](https://github.com/novikd)). +* Fix database iterator waiting code [#60314](https://github.com/ClickHouse/ClickHouse/pull/60314) ([Sergei Trifonov](https://github.com/serxa)). +* QueryCache: Don't acquire the query count mutex if not necessary [#60348](https://github.com/ClickHouse/ClickHouse/pull/60348) ([zhongyuankai](https://github.com/zhongyuankai)). +* Fix bugfix check (due to unknown commit_logs_cache_size_threshold) [#60375](https://github.com/ClickHouse/ClickHouse/pull/60375) ([Azat Khuzhin](https://github.com/azat)). +* Enable testing with `io_uring` back [#60383](https://github.com/ClickHouse/ClickHouse/pull/60383) ([Nikita Taranov](https://github.com/nickitat)). +* Analyzer - improve hiding secret arguments. [#60386](https://github.com/ClickHouse/ClickHouse/pull/60386) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* CI: make workflow yml abstract [#60421](https://github.com/ClickHouse/ClickHouse/pull/60421) ([Max K.](https://github.com/maxknv)). +* Improve test test_reload_clusters_config [#60426](https://github.com/ClickHouse/ClickHouse/pull/60426) ([Kruglov Pavel](https://github.com/Avogar)). +* Revert "Revert "Merge pull request [#56864](https://github.com/ClickHouse/ClickHouse/issues/56864) from ClickHouse/broken-projections-better-handling"" [#60452](https://github.com/ClickHouse/ClickHouse/pull/60452) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Do not check to and from files existence in metadata_storage because it does not see uncommitted changes [#60462](https://github.com/ClickHouse/ClickHouse/pull/60462) ([Alexander Gololobov](https://github.com/davenger)). +* Fix option ambiguous in `clickhouse-local` [#60475](https://github.com/ClickHouse/ClickHouse/pull/60475) ([豪肥肥](https://github.com/HowePa)). +* Fix: test_parallel_replicas_custom_key_load_balancing [#60485](https://github.com/ClickHouse/ClickHouse/pull/60485) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix: progress bar for *Cluster table functions [#60491](https://github.com/ClickHouse/ClickHouse/pull/60491) ([Igor Nikonov](https://github.com/devcrafter)). +* Analyzer: Support different ObjectJSON on shards [#60497](https://github.com/ClickHouse/ClickHouse/pull/60497) ([Dmitry Novik](https://github.com/novikd)). +* Cancel PipelineExecutor properly in case of exception in spawnThreads [#60499](https://github.com/ClickHouse/ClickHouse/pull/60499) ([Kruglov Pavel](https://github.com/Avogar)). +* Refactor StorageSystemOneBlock [#60510](https://github.com/ClickHouse/ClickHouse/pull/60510) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Simple cleanup while fixing progress bar [#60513](https://github.com/ClickHouse/ClickHouse/pull/60513) ([Igor Nikonov](https://github.com/devcrafter)). +* PullingAsyncPipelineExecutor cleanup [#60515](https://github.com/ClickHouse/ClickHouse/pull/60515) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix bad error message [#60518](https://github.com/ClickHouse/ClickHouse/pull/60518) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Synchronize Access [#60519](https://github.com/ClickHouse/ClickHouse/pull/60519) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Synchronize metrics and Keeper [#60520](https://github.com/ClickHouse/ClickHouse/pull/60520) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enforce clang-tidy in `programs/` and `utils/` headers [#60521](https://github.com/ClickHouse/ClickHouse/pull/60521) ([Robert Schulze](https://github.com/rschu1ze)). +* Synchronize parsers [#60522](https://github.com/ClickHouse/ClickHouse/pull/60522) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a bunch of clang-tidy warnings in headers [#60523](https://github.com/ClickHouse/ClickHouse/pull/60523) ([Robert Schulze](https://github.com/rschu1ze)). +* General sanity in function `seriesOutliersDetectTukey` [#60535](https://github.com/ClickHouse/ClickHouse/pull/60535) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Chinese document for max_query_size, max_parser_depth and optimize_functions_to_subcolumns [#60541](https://github.com/ClickHouse/ClickHouse/pull/60541) ([Alex Cheng](https://github.com/Alex-Cheng)). +* Userspace page cache again [#60552](https://github.com/ClickHouse/ClickHouse/pull/60552) ([Michael Kolupaev](https://github.com/al13n321)). +* Traverse shadow directory for system.remote_data_paths [#60585](https://github.com/ClickHouse/ClickHouse/pull/60585) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Add test for [#58906](https://github.com/ClickHouse/ClickHouse/issues/58906) [#60597](https://github.com/ClickHouse/ClickHouse/pull/60597) ([Raúl Marín](https://github.com/Algunenano)). +* Use python zipfile to have x-platform idempotent lambda packages [#60603](https://github.com/ClickHouse/ClickHouse/pull/60603) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* tests: suppress data-race in librdkafka statistics code [#60604](https://github.com/ClickHouse/ClickHouse/pull/60604) ([Azat Khuzhin](https://github.com/azat)). +* Update version after release [#60605](https://github.com/ClickHouse/ClickHouse/pull/60605) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v24.2.1.2248-stable [#60607](https://github.com/ClickHouse/ClickHouse/pull/60607) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Addition to changelog [#60609](https://github.com/ClickHouse/ClickHouse/pull/60609) ([Anton Popov](https://github.com/CurtizJ)). +* internal: Refine rust prql code [#60617](https://github.com/ClickHouse/ClickHouse/pull/60617) ([Maximilian Roos](https://github.com/max-sixty)). +* fix(rust): Fix skim's panic handler [#60621](https://github.com/ClickHouse/ClickHouse/pull/60621) ([Maximilian Roos](https://github.com/max-sixty)). +* Resubmit "Analyzer: compute ALIAS columns right after reading" [#60641](https://github.com/ClickHouse/ClickHouse/pull/60641) ([vdimir](https://github.com/vdimir)). +* Analyzer: Fix bug with join_use_nulls and PREWHERE [#60655](https://github.com/ClickHouse/ClickHouse/pull/60655) ([vdimir](https://github.com/vdimir)). +* Add test for [#59891](https://github.com/ClickHouse/ClickHouse/issues/59891) [#60657](https://github.com/ClickHouse/ClickHouse/pull/60657) ([Raúl Marín](https://github.com/Algunenano)). +* Fix missed entries in system.part_log in case of fetch preferred over merges/mutations [#60659](https://github.com/ClickHouse/ClickHouse/pull/60659) ([Azat Khuzhin](https://github.com/azat)). +* Always apply first minmax index among available skip indices [#60675](https://github.com/ClickHouse/ClickHouse/pull/60675) ([Igor Nikonov](https://github.com/devcrafter)). +* Remove bad test `02152_http_external_tables_memory_tracking` [#60690](https://github.com/ClickHouse/ClickHouse/pull/60690) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix questionable behavior in the `parseDateTimeBestEffort` function. [#60691](https://github.com/ClickHouse/ClickHouse/pull/60691) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky checks [#60694](https://github.com/ClickHouse/ClickHouse/pull/60694) ([Azat Khuzhin](https://github.com/azat)). +* Resubmit http_external_tables_memory_tracking test [#60695](https://github.com/ClickHouse/ClickHouse/pull/60695) ([Azat Khuzhin](https://github.com/azat)). +* Fix bugfix and upgrade checks (due to "Unknown handler type 'redirect'" error) [#60696](https://github.com/ClickHouse/ClickHouse/pull/60696) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_grant_and_revoke/test.py::test_grant_all_on_table (after syncing with cloud) [#60699](https://github.com/ClickHouse/ClickHouse/pull/60699) ([Azat Khuzhin](https://github.com/azat)). +* Remove unit test for ColumnObject [#60709](https://github.com/ClickHouse/ClickHouse/pull/60709) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve unit tests [#60710](https://github.com/ClickHouse/ClickHouse/pull/60710) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix scheduler fairness test [#60712](https://github.com/ClickHouse/ClickHouse/pull/60712) ([Sergei Trifonov](https://github.com/serxa)). +* Do not retry queries if container is down in integration tests (resubmit) [#60714](https://github.com/ClickHouse/ClickHouse/pull/60714) ([Azat Khuzhin](https://github.com/azat)). +* Mark one setting as obsolete [#60715](https://github.com/ClickHouse/ClickHouse/pull/60715) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a test with Analyzer [#60723](https://github.com/ClickHouse/ClickHouse/pull/60723) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Two tests are fixed with Analyzer [#60724](https://github.com/ClickHouse/ClickHouse/pull/60724) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old code [#60728](https://github.com/ClickHouse/ClickHouse/pull/60728) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove more code from LIVE VIEW [#60729](https://github.com/ClickHouse/ClickHouse/pull/60729) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `test_keeper_back_to_back/test.py::test_concurrent_watches` [#60749](https://github.com/ClickHouse/ClickHouse/pull/60749) ([Antonio Andelic](https://github.com/antonio2368)). +* Catch exceptions on finalize in `InterserverIOHTTPHandler` [#60769](https://github.com/ClickHouse/ClickHouse/pull/60769) ([Antonio Andelic](https://github.com/antonio2368)). +* Reduce flakiness of 02932_refreshable_materialized_views [#60771](https://github.com/ClickHouse/ClickHouse/pull/60771) ([Michael Kolupaev](https://github.com/al13n321)). +* Use 64-bit capabilities if available [#60775](https://github.com/ClickHouse/ClickHouse/pull/60775) ([Azat Khuzhin](https://github.com/azat)). +* Include multiline logs in fuzzer fatal.log report [#60796](https://github.com/ClickHouse/ClickHouse/pull/60796) ([Raúl Marín](https://github.com/Algunenano)). +* Add missing clone calls related to compression [#60810](https://github.com/ClickHouse/ClickHouse/pull/60810) ([Raúl Marín](https://github.com/Algunenano)). +* New private runners [#60811](https://github.com/ClickHouse/ClickHouse/pull/60811) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Move userspace page cache settings to the correct section of SettingsChangeHistory.h [#60812](https://github.com/ClickHouse/ClickHouse/pull/60812) ([Michael Kolupaev](https://github.com/al13n321)). +* Update version_date.tsv and changelogs after v23.8.10.43-lts [#60851](https://github.com/ClickHouse/ClickHouse/pull/60851) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Fix fuzzer report [#60853](https://github.com/ClickHouse/ClickHouse/pull/60853) ([Raúl Marín](https://github.com/Algunenano)). +* Update version_date.tsv and changelogs after v23.3.20.27-lts [#60857](https://github.com/ClickHouse/ClickHouse/pull/60857) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Refactor OptimizeDateOrDateTimeConverterWithPreimageVisitor [#60875](https://github.com/ClickHouse/ClickHouse/pull/60875) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* Fix race in PageCache [#60878](https://github.com/ClickHouse/ClickHouse/pull/60878) ([Michael Kolupaev](https://github.com/al13n321)). +* Small changes in async inserts code [#60885](https://github.com/ClickHouse/ClickHouse/pull/60885) ([Nikita Taranov](https://github.com/nickitat)). +* Remove useless verbose logging from AWS library [#60921](https://github.com/ClickHouse/ClickHouse/pull/60921) ([alesapin](https://github.com/alesapin)). +* Throw on query timeout in ZooKeeperRetries [#60922](https://github.com/ClickHouse/ClickHouse/pull/60922) ([Antonio Andelic](https://github.com/antonio2368)). +* Bring clickhouse-test changes from private [#60924](https://github.com/ClickHouse/ClickHouse/pull/60924) ([Raúl Marín](https://github.com/Algunenano)). +* Add debug info to exceptions in `IMergeTreeDataPart::checkConsistency()` [#60981](https://github.com/ClickHouse/ClickHouse/pull/60981) ([Nikita Taranov](https://github.com/nickitat)). +* Fix a typo [#60987](https://github.com/ClickHouse/ClickHouse/pull/60987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace some header includes with forward declarations [#61003](https://github.com/ClickHouse/ClickHouse/pull/61003) ([Amos Bird](https://github.com/amosbird)). +* Speed up cctools building [#61011](https://github.com/ClickHouse/ClickHouse/pull/61011) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix ASTRenameQuery::clone [#61013](https://github.com/ClickHouse/ClickHouse/pull/61013) ([vdimir](https://github.com/vdimir)). +* Update README.md [#61021](https://github.com/ClickHouse/ClickHouse/pull/61021) ([Tyler Hannan](https://github.com/tylerhannan)). +* Fix TableFunctionExecutable::skipAnalysisForArguments [#61037](https://github.com/ClickHouse/ClickHouse/pull/61037) ([Dmitry Novik](https://github.com/novikd)). +* Fix: parallel replicas with PREWHERE (ubsan) [#61052](https://github.com/ClickHouse/ClickHouse/pull/61052) ([Igor Nikonov](https://github.com/devcrafter)). +* Fast fix tests. [#61056](https://github.com/ClickHouse/ClickHouse/pull/61056) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `test_placement_info` [#61057](https://github.com/ClickHouse/ClickHouse/pull/61057) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Fix: parallel replicas with CTEs, crash in EXPLAIN SYNTAX with analyzer [#61059](https://github.com/ClickHouse/ClickHouse/pull/61059) ([Igor Nikonov](https://github.com/devcrafter)). +* Debug fuzzer failures [#61062](https://github.com/ClickHouse/ClickHouse/pull/61062) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add regression tests for fixed issues [#61076](https://github.com/ClickHouse/ClickHouse/pull/61076) ([Antonio Andelic](https://github.com/antonio2368)). +* Analyzer: Fix 01244_optimize_distributed_group_by_sharding_key [#61089](https://github.com/ClickHouse/ClickHouse/pull/61089) ([Dmitry Novik](https://github.com/novikd)). +* Use global scalars cache with analyzer [#61104](https://github.com/ClickHouse/ClickHouse/pull/61104) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix removing is_active node after re-creation [#61105](https://github.com/ClickHouse/ClickHouse/pull/61105) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update 02962_system_sync_replica_lightweight_from_modifier.sh [#61110](https://github.com/ClickHouse/ClickHouse/pull/61110) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Simplify bridges [#61118](https://github.com/ClickHouse/ClickHouse/pull/61118) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update cppkafka to v0.4.1 [#61119](https://github.com/ClickHouse/ClickHouse/pull/61119) ([Ilya Golshtein](https://github.com/ilejn)). +* CI: add wf class in ci_config [#61122](https://github.com/ClickHouse/ClickHouse/pull/61122) ([Max K.](https://github.com/maxknv)). +* QueryFuzzer: replace element randomly when AST part buffer is full [#61124](https://github.com/ClickHouse/ClickHouse/pull/61124) ([Tomer Shafir](https://github.com/tomershafir)). +* CI: make style check fast [#61125](https://github.com/ClickHouse/ClickHouse/pull/61125) ([Max K.](https://github.com/maxknv)). +* Better gitignore [#61128](https://github.com/ClickHouse/ClickHouse/pull/61128) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix something strange [#61129](https://github.com/ClickHouse/ClickHouse/pull/61129) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update check-large-objects.sh to be language neutral [#61130](https://github.com/ClickHouse/ClickHouse/pull/61130) ([Dan Wu](https://github.com/wudanzy)). +* Throw memory limit exceptions to avoid OOM in some places [#61132](https://github.com/ClickHouse/ClickHouse/pull/61132) ([alesapin](https://github.com/alesapin)). +* Fix test_distributed_directory_monitor_split_batch_on_failure flakienss [#61136](https://github.com/ClickHouse/ClickHouse/pull/61136) ([Azat Khuzhin](https://github.com/azat)). +* Fix llvm symbolizer on CI [#61147](https://github.com/ClickHouse/ClickHouse/pull/61147) ([Azat Khuzhin](https://github.com/azat)). +* Some clang-tidy fixes [#61150](https://github.com/ClickHouse/ClickHouse/pull/61150) ([Robert Schulze](https://github.com/rschu1ze)). +* Revive "Less contention in the cache, part 2" [#61152](https://github.com/ClickHouse/ClickHouse/pull/61152) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Enable black back [#61159](https://github.com/ClickHouse/ClickHouse/pull/61159) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* CI: fix nightly job issue [#61160](https://github.com/ClickHouse/ClickHouse/pull/61160) ([Max K.](https://github.com/maxknv)). +* Split `RangeHashedDictionary` [#61162](https://github.com/ClickHouse/ClickHouse/pull/61162) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Remove a few templates from Aggregator.cpp [#61171](https://github.com/ClickHouse/ClickHouse/pull/61171) ([Raúl Marín](https://github.com/Algunenano)). +* Avoid some logical errors in experimantal Object type [#61173](https://github.com/ClickHouse/ClickHouse/pull/61173) ([Kruglov Pavel](https://github.com/Avogar)). +* Update ReadSettings.h [#61174](https://github.com/ClickHouse/ClickHouse/pull/61174) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CI: ARM integration tests: disable tests with HDFS [#61182](https://github.com/ClickHouse/ClickHouse/pull/61182) ([Max K.](https://github.com/maxknv)). +* Disable sanitizers with 02784_parallel_replicas_automatic_decision_join [#61184](https://github.com/ClickHouse/ClickHouse/pull/61184) ([Raúl Marín](https://github.com/Algunenano)). +* Fix `02887_mutations_subcolumns` test flakiness [#61198](https://github.com/ClickHouse/ClickHouse/pull/61198) ([Nikita Taranov](https://github.com/nickitat)). +* Make variant tests a bit faster [#61199](https://github.com/ClickHouse/ClickHouse/pull/61199) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix strange log message [#61206](https://github.com/ClickHouse/ClickHouse/pull/61206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01603_insert_select_too_many_parts flakiness [#61218](https://github.com/ClickHouse/ClickHouse/pull/61218) ([Azat Khuzhin](https://github.com/azat)). +* Make every style-checker runner types scaling-out very quickly [#61231](https://github.com/ClickHouse/ClickHouse/pull/61231) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Improve `test_failed_mutations` [#61235](https://github.com/ClickHouse/ClickHouse/pull/61235) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Fix `test_merge_tree_load_parts/test.py::test_merge_tree_load_parts_corrupted` [#61236](https://github.com/ClickHouse/ClickHouse/pull/61236) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* fix `forget_partition` test [#61237](https://github.com/ClickHouse/ClickHouse/pull/61237) ([Sergei Trifonov](https://github.com/serxa)). +* Print more info in `02572_system_logs_materialized_views_ignore_errors` to debug [#61246](https://github.com/ClickHouse/ClickHouse/pull/61246) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Fix runtime error in AST Fuzzer [#61248](https://github.com/ClickHouse/ClickHouse/pull/61248) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Add retries to `02908_many_requests_to_system_replicas` [#61253](https://github.com/ClickHouse/ClickHouse/pull/61253) ([Nikita Taranov](https://github.com/nickitat)). +* Followup fix ASTRenameQuery::clone [#61254](https://github.com/ClickHouse/ClickHouse/pull/61254) ([vdimir](https://github.com/vdimir)). +* Disable test 02998_primary_key_skip_columns.sql in sanitizer builds as it can be slow [#61256](https://github.com/ClickHouse/ClickHouse/pull/61256) ([Kruglov Pavel](https://github.com/Avogar)). +* Update curl to curl with data race fix [#61264](https://github.com/ClickHouse/ClickHouse/pull/61264) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Fix `01417_freeze_partition_verbose` [#61266](https://github.com/ClickHouse/ClickHouse/pull/61266) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Free memory earlier in inserts [#61267](https://github.com/ClickHouse/ClickHouse/pull/61267) ([Anton Popov](https://github.com/CurtizJ)). +* Fixing test_build_sets_from_multiple_threads/test.py::test_set [#61286](https://github.com/ClickHouse/ClickHouse/pull/61286) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Analyzer: Fix virtual columns in StorageMerge [#61298](https://github.com/ClickHouse/ClickHouse/pull/61298) ([Dmitry Novik](https://github.com/novikd)). +* Fix 01952_optimize_distributed_group_by_sharding_key with analyzer. [#61301](https://github.com/ClickHouse/ClickHouse/pull/61301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fix data race in poco tcp server [#61309](https://github.com/ClickHouse/ClickHouse/pull/61309) ([Sema Checherinda](https://github.com/CheSema)). +* Don't use default cluster in test test_distibuted_settings [#61314](https://github.com/ClickHouse/ClickHouse/pull/61314) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix false positive assertion in cache [#61319](https://github.com/ClickHouse/ClickHouse/pull/61319) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix test test_input_format_parallel_parsing_memory_tracking [#61322](https://github.com/ClickHouse/ClickHouse/pull/61322) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix 01761_cast_to_enum_nullable with analyzer. [#61323](https://github.com/ClickHouse/ClickHouse/pull/61323) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add zookeeper retries for exists check in forcefullyRemoveBrokenOutdatedPartFromZooKeeper [#61324](https://github.com/ClickHouse/ClickHouse/pull/61324) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor changes in stress and fuzzer reports [#61333](https://github.com/ClickHouse/ClickHouse/pull/61333) ([Raúl Marín](https://github.com/Algunenano)). +* Un-flake `test_undrop_query` [#61348](https://github.com/ClickHouse/ClickHouse/pull/61348) ([Robert Schulze](https://github.com/rschu1ze)). +* Tiny improvement for replication.lib [#61361](https://github.com/ClickHouse/ClickHouse/pull/61361) ([alesapin](https://github.com/alesapin)). +* Fix bugfix check (due to "unknown object storage type: azure") [#61363](https://github.com/ClickHouse/ClickHouse/pull/61363) ([Azat Khuzhin](https://github.com/azat)). +* Fix `01599_multiline_input_and_singleline_comments` 3 minute wait [#61371](https://github.com/ClickHouse/ClickHouse/pull/61371) ([Sergei Trifonov](https://github.com/serxa)). +* Terminate EC2 on spot event if runner isn't running [#61377](https://github.com/ClickHouse/ClickHouse/pull/61377) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try fix docs check [#61378](https://github.com/ClickHouse/ClickHouse/pull/61378) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix `heap-use-after-free` for Merge table with alias [#61380](https://github.com/ClickHouse/ClickHouse/pull/61380) ([Antonio Andelic](https://github.com/antonio2368)). +* Disable `optimize_rewrite_sum_if_to_count_if` if return type is nullable (new analyzer) [#61389](https://github.com/ClickHouse/ClickHouse/pull/61389) ([Antonio Andelic](https://github.com/antonio2368)). +* Analyzer: Fix planner context for subquery in StorageMerge [#61392](https://github.com/ClickHouse/ClickHouse/pull/61392) ([Dmitry Novik](https://github.com/novikd)). +* Fix `test_failed_async_inserts` [#61394](https://github.com/ClickHouse/ClickHouse/pull/61394) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix test test_system_clusters_actual_information flakiness [#61395](https://github.com/ClickHouse/ClickHouse/pull/61395) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove default cluster from default config from test config [#61396](https://github.com/ClickHouse/ClickHouse/pull/61396) ([Raúl Marín](https://github.com/Algunenano)). +* Enable clang-tidy in headers [#61406](https://github.com/ClickHouse/ClickHouse/pull/61406) ([Robert Schulze](https://github.com/rschu1ze)). +* Add sanity check for poll_max_batch_size FileLog setting [#61408](https://github.com/ClickHouse/ClickHouse/pull/61408) ([Kruglov Pavel](https://github.com/Avogar)). +* ThreadFuzzer: randomize sleep time [#61410](https://github.com/ClickHouse/ClickHouse/pull/61410) ([Tomer Shafir](https://github.com/tomershafir)). +* Update version_date.tsv and changelogs after v23.8.11.28-lts [#61416](https://github.com/ClickHouse/ClickHouse/pull/61416) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.3.21.26-lts [#61418](https://github.com/ClickHouse/ClickHouse/pull/61418) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v24.1.7.18-stable [#61419](https://github.com/ClickHouse/ClickHouse/pull/61419) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v24.2.2.71-stable [#61420](https://github.com/ClickHouse/ClickHouse/pull/61420) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.12.5.81-stable [#61421](https://github.com/ClickHouse/ClickHouse/pull/61421) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Restore automerge for approved PRs [#61433](https://github.com/ClickHouse/ClickHouse/pull/61433) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable broken SonarCloud [#61434](https://github.com/ClickHouse/ClickHouse/pull/61434) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix `01599_multiline_input_and_singleline_comments` properly [#61440](https://github.com/ClickHouse/ClickHouse/pull/61440) ([Sergei Trifonov](https://github.com/serxa)). +* Convert test 02998_system_dns_cache_table to smoke and mirrors [#61443](https://github.com/ClickHouse/ClickHouse/pull/61443) ([vdimir](https://github.com/vdimir)). +* Check boundaries for some settings in parallel replicas [#61455](https://github.com/ClickHouse/ClickHouse/pull/61455) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use SHARD_LOAD_QUEUE_BACKLOG for dictionaries in tests [#61462](https://github.com/ClickHouse/ClickHouse/pull/61462) ([vdimir](https://github.com/vdimir)). +* Split `02125_lz4_compression_bug` [#61465](https://github.com/ClickHouse/ClickHouse/pull/61465) ([Antonio Andelic](https://github.com/antonio2368)). +* Correctly process last stacktrace in `postprocess-traces.pl` [#61470](https://github.com/ClickHouse/ClickHouse/pull/61470) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix test `test_polymorphic_parts` [#61477](https://github.com/ClickHouse/ClickHouse/pull/61477) ([Anton Popov](https://github.com/CurtizJ)). +* A definitive guide to CAST [#61491](https://github.com/ClickHouse/ClickHouse/pull/61491) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor rename in FileCache [#61494](https://github.com/ClickHouse/ClickHouse/pull/61494) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove useless code [#61498](https://github.com/ClickHouse/ClickHouse/pull/61498) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fuzzers [#61499](https://github.com/ClickHouse/ClickHouse/pull/61499) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update jdbc.md [#61506](https://github.com/ClickHouse/ClickHouse/pull/61506) ([San](https://github.com/santrancisco)). +* Fix error in clickhouse-client [#61507](https://github.com/ClickHouse/ClickHouse/pull/61507) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix clang-tidy build [#61519](https://github.com/ClickHouse/ClickHouse/pull/61519) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix infinite loop in function `hop` [#61523](https://github.com/ClickHouse/ClickHouse/pull/61523) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve tests 00159_parallel_formatting_* to to avoid timeouts [#61532](https://github.com/ClickHouse/ClickHouse/pull/61532) ([Kruglov Pavel](https://github.com/Avogar)). +* Refactoring of reading from compact parts [#61535](https://github.com/ClickHouse/ClickHouse/pull/61535) ([Anton Popov](https://github.com/CurtizJ)). +* Don't run 01459_manual_write_to_replicas in debug build as it's too slow [#61538](https://github.com/ClickHouse/ClickHouse/pull/61538) ([Kruglov Pavel](https://github.com/Avogar)). +* CI: ARM integration test - skip hdfs, kerberos, kafka [#61542](https://github.com/ClickHouse/ClickHouse/pull/61542) ([Max K.](https://github.com/maxknv)). +* More logging for loading of tables [#61546](https://github.com/ClickHouse/ClickHouse/pull/61546) ([Sergei Trifonov](https://github.com/serxa)). +* Fixing 01584_distributed_buffer_cannot_find_column with analyzer. [#61550](https://github.com/ClickHouse/ClickHouse/pull/61550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Wait for done mutation with more logs and asserts [#61554](https://github.com/ClickHouse/ClickHouse/pull/61554) ([alesapin](https://github.com/alesapin)). +* Fix read_rows count with external group by [#61555](https://github.com/ClickHouse/ClickHouse/pull/61555) ([Alexander Tokmakov](https://github.com/tavplubix)). +* queries-file should be used to specify file [#61557](https://github.com/ClickHouse/ClickHouse/pull/61557) ([danila-ermakov](https://github.com/danila-ermakov)). +* Fix `02481_async_insert_dedup_token` [#61568](https://github.com/ClickHouse/ClickHouse/pull/61568) ([Antonio Andelic](https://github.com/antonio2368)). +* Add a comment after [#61458](https://github.com/ClickHouse/ClickHouse/issues/61458) [#61580](https://github.com/ClickHouse/ClickHouse/pull/61580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix clickhouse-test client option and CLICKHOUSE_URL_PARAMS interference [#61596](https://github.com/ClickHouse/ClickHouse/pull/61596) ([vdimir](https://github.com/vdimir)). +* CI: remove compose files from integration test docker [#61597](https://github.com/ClickHouse/ClickHouse/pull/61597) ([Max K.](https://github.com/maxknv)). +* Fix 01244_optimize_distributed_group_by_sharding_key by ordering output [#61602](https://github.com/ClickHouse/ClickHouse/pull/61602) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove some tests from analyzer_tech_debt [#61603](https://github.com/ClickHouse/ClickHouse/pull/61603) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reduce header dependencies [#61604](https://github.com/ClickHouse/ClickHouse/pull/61604) ([Raúl Marín](https://github.com/Algunenano)). +* Remove some magic_enum from headers [#61606](https://github.com/ClickHouse/ClickHouse/pull/61606) ([Raúl Marín](https://github.com/Algunenano)). +* Fix configs for upgrade and bugfix [#61607](https://github.com/ClickHouse/ClickHouse/pull/61607) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add tests for multiple fuzzer issues [#61614](https://github.com/ClickHouse/ClickHouse/pull/61614) ([Raúl Marín](https://github.com/Algunenano)). +* Try to fix `02908_many_requests_to_system_replicas` again [#61616](https://github.com/ClickHouse/ClickHouse/pull/61616) ([Nikita Taranov](https://github.com/nickitat)). +* Verbose error message about analyzer_compatibility_join_using_top_level_identifier [#61631](https://github.com/ClickHouse/ClickHouse/pull/61631) ([vdimir](https://github.com/vdimir)). +* Fix 00223_shard_distributed_aggregation_memory_efficient with analyzer [#61649](https://github.com/ClickHouse/ClickHouse/pull/61649) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better fuzzer logs [#61650](https://github.com/ClickHouse/ClickHouse/pull/61650) ([Raúl Marín](https://github.com/Algunenano)). +* Fix flaky `02122_parallel_formatting_Template` [#61651](https://github.com/ClickHouse/ClickHouse/pull/61651) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix Aggregator when data is empty [#61654](https://github.com/ClickHouse/ClickHouse/pull/61654) ([Antonio Andelic](https://github.com/antonio2368)). +* Restore poco SUN files [#61655](https://github.com/ClickHouse/ClickHouse/pull/61655) ([Andy Fiddaman](https://github.com/citrus-it)). +* Another fix for `SumIfToCountIfPass` [#61656](https://github.com/ClickHouse/ClickHouse/pull/61656) ([Antonio Andelic](https://github.com/antonio2368)). +* Keeper: fix data race during snapshot destructor call [#61657](https://github.com/ClickHouse/ClickHouse/pull/61657) ([Antonio Andelic](https://github.com/antonio2368)). +* CI: integration tests: use runner as py module [#61658](https://github.com/ClickHouse/ClickHouse/pull/61658) ([Max K.](https://github.com/maxknv)). +* Fix logging of autoscaling lambda, add test for effective_capacity [#61662](https://github.com/ClickHouse/ClickHouse/pull/61662) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Small change in `DatabaseOnDisk::iterateMetadataFiles()` [#61664](https://github.com/ClickHouse/ClickHouse/pull/61664) ([Nikita Taranov](https://github.com/nickitat)). +* Build improvements by removing magic enum from header and apply some explicit template instantiation [#61665](https://github.com/ClickHouse/ClickHouse/pull/61665) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Update the dictionary for OSSFuzz [#61672](https://github.com/ClickHouse/ClickHouse/pull/61672) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Inhibit randomization in some tests and exclude some long tests from debug runs [#61676](https://github.com/ClickHouse/ClickHouse/pull/61676) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#61669](https://github.com/ClickHouse/ClickHouse/issues/61669) [#61678](https://github.com/ClickHouse/ClickHouse/pull/61678) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix use-of-uninitialized-value in HedgedConnections [#61679](https://github.com/ClickHouse/ClickHouse/pull/61679) ([Nikolay Degterinsky](https://github.com/evillique)). +* Remove clickhouse-diagnostics from the package [#61681](https://github.com/ClickHouse/ClickHouse/pull/61681) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix use-of-uninitialized-value in parseDateTimeBestEffort [#61694](https://github.com/ClickHouse/ClickHouse/pull/61694) ([Nikolay Degterinsky](https://github.com/evillique)). +* poco foundation: add illumos support [#61701](https://github.com/ClickHouse/ClickHouse/pull/61701) ([Andy Fiddaman](https://github.com/citrus-it)). +* contrib/c-ares: add illumos as a platform [#61702](https://github.com/ClickHouse/ClickHouse/pull/61702) ([Andy Fiddaman](https://github.com/citrus-it)). +* contrib/curl: Add illumos support [#61704](https://github.com/ClickHouse/ClickHouse/pull/61704) ([Andy Fiddaman](https://github.com/citrus-it)). +* Fuzzer: Try a different way to wait for the server [#61706](https://github.com/ClickHouse/ClickHouse/pull/61706) ([Raúl Marín](https://github.com/Algunenano)). +* Disable some tests for SMT [#61708](https://github.com/ClickHouse/ClickHouse/pull/61708) ([Raúl Marín](https://github.com/Algunenano)). +* Fix signal handler for sanitizer signals [#61709](https://github.com/ClickHouse/ClickHouse/pull/61709) ([Antonio Andelic](https://github.com/antonio2368)). +* Avoid `IsADirectoryError: Is a directory contrib/azure` [#61710](https://github.com/ClickHouse/ClickHouse/pull/61710) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Analyzer: fix group_by_use_nulls [#61717](https://github.com/ClickHouse/ClickHouse/pull/61717) ([Dmitry Novik](https://github.com/novikd)). +* Analyzer: Clear list of broken integration tests [#61718](https://github.com/ClickHouse/ClickHouse/pull/61718) ([Dmitry Novik](https://github.com/novikd)). +* CI: modify CI from PR body [#61725](https://github.com/ClickHouse/ClickHouse/pull/61725) ([Max K.](https://github.com/maxknv)). +* Add test for [#57820](https://github.com/ClickHouse/ClickHouse/issues/57820) [#61726](https://github.com/ClickHouse/ClickHouse/pull/61726) ([Dmitry Novik](https://github.com/novikd)). +* Revert "Revert "Un-flake test_undrop_query"" [#61727](https://github.com/ClickHouse/ClickHouse/pull/61727) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* FunctionsConversion: Start simplifying templates [#61733](https://github.com/ClickHouse/ClickHouse/pull/61733) ([Raúl Marín](https://github.com/Algunenano)). +* CI: modify it [#61735](https://github.com/ClickHouse/ClickHouse/pull/61735) ([Max K.](https://github.com/maxknv)). +* Fix segfault in SquashingTransform [#61736](https://github.com/ClickHouse/ClickHouse/pull/61736) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix DWARF format failing to skip DW_FORM_strx3 attributes [#61737](https://github.com/ClickHouse/ClickHouse/pull/61737) ([Michael Kolupaev](https://github.com/al13n321)). +* There is no such thing as broken tests [#61739](https://github.com/ClickHouse/ClickHouse/pull/61739) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Process removed files, decouple _check_mime [#61751](https://github.com/ClickHouse/ClickHouse/pull/61751) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Keeper fix: destroy `KeeperDispatcher` first [#61752](https://github.com/ClickHouse/ClickHouse/pull/61752) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix flaky `03014_async_with_dedup_part_log_rmt` [#61757](https://github.com/ClickHouse/ClickHouse/pull/61757) ([Antonio Andelic](https://github.com/antonio2368)). +* FunctionsConversion: Remove another batch of bad templates [#61773](https://github.com/ClickHouse/ClickHouse/pull/61773) ([Raúl Marín](https://github.com/Algunenano)). +* Revert "Fix bug when reading system.parts using UUID (issue 61220)." [#61774](https://github.com/ClickHouse/ClickHouse/pull/61774) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* CI: disable grpc tests on ARM [#61778](https://github.com/ClickHouse/ClickHouse/pull/61778) ([Max K.](https://github.com/maxknv)). +* Fix more tests with virtual columns in StorageMerge. [#61787](https://github.com/ClickHouse/ClickHouse/pull/61787) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove already not flaky tests with analyzer. [#61788](https://github.com/ClickHouse/ClickHouse/pull/61788) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Analyzer: Fix assert in JOIN with Distributed table [#61789](https://github.com/ClickHouse/ClickHouse/pull/61789) ([vdimir](https://github.com/vdimir)). +* A test can be slow in debug build [#61796](https://github.com/ClickHouse/ClickHouse/pull/61796) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Updated clang-19 to master. [#61798](https://github.com/ClickHouse/ClickHouse/pull/61798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test "00002_log_and_exception_messages_formatting" [#61821](https://github.com/ClickHouse/ClickHouse/pull/61821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A test is too slow for debug [#61822](https://github.com/ClickHouse/ClickHouse/pull/61822) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove DataStreams [#61824](https://github.com/ClickHouse/ClickHouse/pull/61824) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better message for logging errors [#61827](https://github.com/ClickHouse/ClickHouse/pull/61827) ([Azat Khuzhin](https://github.com/azat)). +* Fix sanitizers suppressions [#61828](https://github.com/ClickHouse/ClickHouse/pull/61828) ([Azat Khuzhin](https://github.com/azat)). +* Remove unused code [#61830](https://github.com/ClickHouse/ClickHouse/pull/61830) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove DataStreams (2) [#61831](https://github.com/ClickHouse/ClickHouse/pull/61831) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update xxhash to v0.8.2 [#61838](https://github.com/ClickHouse/ClickHouse/pull/61838) ([Shubham Ranjan](https://github.com/shubhamranjan)). +* Fix: DISTINCT in subquery with analyzer [#61847](https://github.com/ClickHouse/ClickHouse/pull/61847) ([Igor Nikonov](https://github.com/devcrafter)). +* Analyzer: fix limit/offset on shards [#61849](https://github.com/ClickHouse/ClickHouse/pull/61849) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Remove PoolBase::AllocateNewBypassingPool [#61866](https://github.com/ClickHouse/ClickHouse/pull/61866) ([Azat Khuzhin](https://github.com/azat)). +* Try to fix 02901_parallel_replicas_rollup with analyzer. [#61875](https://github.com/ClickHouse/ClickHouse/pull/61875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add test for [#57808](https://github.com/ClickHouse/ClickHouse/issues/57808) [#61879](https://github.com/ClickHouse/ClickHouse/pull/61879) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* CI: merge queue support [#61881](https://github.com/ClickHouse/ClickHouse/pull/61881) ([Max K.](https://github.com/maxknv)). +* Update create.sql [#61885](https://github.com/ClickHouse/ClickHouse/pull/61885) ([Kseniia Sumarokova](https://github.com/kssenii)). +* no smaller unit in date_trunc [#61888](https://github.com/ClickHouse/ClickHouse/pull/61888) ([jsc0218](https://github.com/jsc0218)). +* Move KQL trash where it is supposed to be [#61903](https://github.com/ClickHouse/ClickHouse/pull/61903) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Changelog for 24.3 [#61909](https://github.com/ClickHouse/ClickHouse/pull/61909) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v23.3.22.3-lts [#61914](https://github.com/ClickHouse/ClickHouse/pull/61914) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.8.12.13-lts [#61915](https://github.com/ClickHouse/ClickHouse/pull/61915) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* No "please" [#61916](https://github.com/ClickHouse/ClickHouse/pull/61916) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v23.12.6.19-stable [#61917](https://github.com/ClickHouse/ClickHouse/pull/61917) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v24.1.8.22-stable [#61918](https://github.com/ClickHouse/ClickHouse/pull/61918) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Fix flaky test_broken_projestions/test.py::test_broken_ignored_replic… [#61932](https://github.com/ClickHouse/ClickHouse/pull/61932) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Check is Rust avaiable for build, if not, suggest a way to disable Rust support [#61938](https://github.com/ClickHouse/ClickHouse/pull/61938) ([Azat Khuzhin](https://github.com/azat)). +* CI: new ci menu in PR body [#61948](https://github.com/ClickHouse/ClickHouse/pull/61948) ([Max K.](https://github.com/maxknv)). +* Remove flaky test `01193_metadata_loading` [#61961](https://github.com/ClickHouse/ClickHouse/pull/61961) ([Nikita Taranov](https://github.com/nickitat)). + +#### Packaging Improvement + +* Adding the `--now` option to enable and start service automatically when installing the database server completely. [#60656](https://github.com/ClickHouse/ClickHouse/pull/60656) ([Chun-Sheng, Li](https://github.com/peter279k)). + diff --git a/docs/changelogs/v24.3.2.23-lts.md b/docs/changelogs/v24.3.2.23-lts.md new file mode 100644 index 00000000000..4d59a1cedf6 --- /dev/null +++ b/docs/changelogs/v24.3.2.23-lts.md @@ -0,0 +1,29 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.3.2.23-lts (8b7d910960c) FIXME as compared to v24.3.1.2672-lts (2c5c589a882) + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix logical error in group_by_use_nulls + grouping set + analyzer + materialize/constant [#61567](https://github.com/ClickHouse/ClickHouse/pull/61567) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix external table cannot parse data type Bool [#62115](https://github.com/ClickHouse/ClickHouse/pull/62115) ([Duc Canh Le](https://github.com/canhld94)). +* Revert "Merge pull request [#61564](https://github.com/ClickHouse/ClickHouse/issues/61564) from liuneng1994/optimize_in_single_value" [#62135](https://github.com/ClickHouse/ClickHouse/pull/62135) ([Raúl Marín](https://github.com/Algunenano)). + +#### CI Fix or Improvement (changelog entry is not required) + +* Backported in [#62030](https://github.com/ClickHouse/ClickHouse/issues/62030):. [#61869](https://github.com/ClickHouse/ClickHouse/pull/61869) ([Nikita Fomichev](https://github.com/fm4v)). +* Backported in [#62057](https://github.com/ClickHouse/ClickHouse/issues/62057): ... [#62044](https://github.com/ClickHouse/ClickHouse/pull/62044) ([Max K.](https://github.com/maxknv)). +* Backported in [#62204](https://github.com/ClickHouse/ClickHouse/issues/62204):. [#62190](https://github.com/ClickHouse/ClickHouse/pull/62190) ([Konstantin Bogdanov](https://github.com/thevar1able)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix some crashes with analyzer and group_by_use_nulls. [#61933](https://github.com/ClickHouse/ClickHouse/pull/61933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix scalars create as select [#61998](https://github.com/ClickHouse/ClickHouse/pull/61998) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Ignore IfChainToMultiIfPass if returned type changed. [#62059](https://github.com/ClickHouse/ClickHouse/pull/62059) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix type for ConvertInToEqualPass [#62066](https://github.com/ClickHouse/ClickHouse/pull/62066) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Revert output Pretty in tty [#62090](https://github.com/ClickHouse/ClickHouse/pull/62090) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/en/development/build-osx.md b/docs/en/development/build-osx.md index 21b9446aa66..a6c49f1f476 100644 --- a/docs/en/development/build-osx.md +++ b/docs/en/development/build-osx.md @@ -55,9 +55,7 @@ To build using Homebrew's vanilla Clang compiler (the only **recommended** way): cd ClickHouse mkdir build export PATH=$(brew --prefix llvm)/bin:$PATH -export CC=$(brew --prefix llvm)/bin/clang -export CXX=$(brew --prefix llvm)/bin/clang++ -cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S . -B build +cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -S . -B build cmake --build build # The resulting binary will be created at: build/programs/clickhouse ``` diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index e08096d8042..42c7e5ac295 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -153,6 +153,26 @@ Upon successful build you get an executable file `ClickHouse//program ls -l programs/clickhouse +### Advanced Building Process {#advanced-building-process} + +#### Minimal Build {#minimal-build} + +If you are not interested in functionality provided by third-party libraries, you can further speed up the build using `cmake` options + +``` +cmake -DENABLE_LIBRARIES=OFF +``` + +In case of problems with any of the development options, you are on your own! + +#### Rust support {#rust-support} + +Rust requires internet connection, in case you don't have it, you can disable Rust support: + +``` +cmake -DENABLE_RUST=OFF +``` + ## Running the Built Executable of ClickHouse {#running-the-built-executable-of-clickhouse} To run the server under the current user you need to navigate to `ClickHouse/programs/server/` (located outside of `build`) and run: @@ -250,10 +270,3 @@ Most probably some of the builds will fail at first times. This is due to the fa You can use GitHub integrated code browser [here](https://github.dev/ClickHouse/ClickHouse). Also, you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual. - -If you are not interested in functionality provided by third-party libraries, you can further speed up the build using `cmake` options -``` --DENABLE_LIBRARIES=0 -``` - -In case of problems with any of the development options, you are on your own! diff --git a/docs/en/engines/database-engines/mysql.md b/docs/en/engines/database-engines/mysql.md index 20434ad124e..ac0ea5e5af1 100644 --- a/docs/en/engines/database-engines/mysql.md +++ b/docs/en/engines/database-engines/mysql.md @@ -4,7 +4,11 @@ sidebar_position: 50 sidebar_label: MySQL --- -# MySQL +import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; + +# MySQL Database Engine + + Allows to connect to databases on a remote MySQL server and perform `INSERT` and `SELECT` queries to exchange data between ClickHouse and MySQL. diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md index 87d59a9394b..e9d0a43242b 100644 --- a/docs/en/engines/table-engines/integrations/mysql.md +++ b/docs/en/engines/table-engines/integrations/mysql.md @@ -4,7 +4,11 @@ sidebar_position: 138 sidebar_label: MySQL --- -# MySQL +import CloudAvailableBadge from '@theme/badges/CloudAvailableBadge'; + +# MySQL Table Engine + + The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server. diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index 131df1a435b..9cc4b11243e 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -8,6 +8,10 @@ sidebar_label: PostgreSQL The PostgreSQL engine allows to perform `SELECT` and `INSERT` queries on data that is stored on a remote PostgreSQL server. +:::note +Currently, only PostgreSQL versions 12 and up are supported. +::: + ## Creating a Table {#creating-a-table} ``` sql diff --git a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md index 62191d9b5e4..7a449f400fd 100644 --- a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -68,6 +68,12 @@ In the results of `SELECT` query, the values of `AggregateFunction` type have im ## Example of an Aggregated Materialized View {#example-of-an-aggregated-materialized-view} +The following examples assumes that you have a database named `test` so make sure you create that if it doesn't already exist: + +```sql +CREATE DATABASE test; +``` + We will create the table `test.visits` that contain the raw data: ``` sql @@ -80,17 +86,24 @@ CREATE TABLE test.visits ) ENGINE = MergeTree ORDER BY (StartDate, CounterID); ``` +Next, we need to create an `AggregatingMergeTree` table that will store `AggregationFunction`s that keep track of the total number of visits and the number of unique users. + `AggregatingMergeTree` materialized view that watches the `test.visits` table, and use the `AggregateFunction` type: ``` sql -CREATE MATERIALIZED VIEW test.mv_visits -( +CREATE TABLE test.agg_visits ( StartDate DateTime64 NOT NULL, CounterID UInt64, Visits AggregateFunction(sum, Nullable(Int32)), Users AggregateFunction(uniq, Nullable(Int32)) ) -ENGINE = AggregatingMergeTree() ORDER BY (StartDate, CounterID) +ENGINE = AggregatingMergeTree() ORDER BY (StartDate, CounterID); +``` + +And then let's create a materialized view that populates `test.agg_visits` from `test.visits` : + +```sql +CREATE MATERIALIZED VIEW test.visits_mv TO test.agg_visits AS SELECT StartDate, CounterID, @@ -104,25 +117,45 @@ Inserting data into the `test.visits` table. ``` sql INSERT INTO test.visits (StartDate, CounterID, Sign, UserID) - VALUES (1667446031, 1, 3, 4) -INSERT INTO test.visits (StartDate, CounterID, Sign, UserID) - VALUES (1667446031, 1, 6, 3) + VALUES (1667446031000, 1, 3, 4), (1667446031000, 1, 6, 3); ``` -The data is inserted in both the table and the materialized view `test.mv_visits`. +The data is inserted in both `test.visits` and `test.agg_visits`. To get the aggregated data, we need to execute a query such as `SELECT ... GROUP BY ...` from the materialized view `test.mv_visits`: -``` sql +```sql SELECT StartDate, sumMerge(Visits) AS Visits, uniqMerge(Users) AS Users -FROM test.mv_visits +FROM test.agg_visits GROUP BY StartDate ORDER BY StartDate; ``` +```text +┌───────────────StartDate─┬─Visits─┬─Users─┐ +│ 2022-11-03 03:27:11.000 │ 9 │ 2 │ +└─────────────────────────┴────────┴───────┘ +``` + +And how about if we add another couple of records to `test.visits`, but this time we'll use a different timestamp for one of the records: + +```sql +INSERT INTO test.visits (StartDate, CounterID, Sign, UserID) + VALUES (1669446031000, 2, 5, 10), (1667446031000, 3, 7, 5); +``` + +If we then run the `SELECT` query again, we'll see the following output: + +```text +┌───────────────StartDate─┬─Visits─┬─Users─┐ +│ 2022-11-03 03:27:11.000 │ 16 │ 3 │ +│ 2022-11-26 07:00:31.000 │ 5 │ 1 │ +└─────────────────────────┴────────┴───────┘ +``` + ## Related Content - Blog: [Using Aggregate Combinators in ClickHouse](https://clickhouse.com/blog/aggregate-functions-combinators-in-clickhouse-for-arrays-maps-and-states) diff --git a/docs/en/engines/table-engines/special/memory.md b/docs/en/engines/table-engines/special/memory.md index 19b5c798a76..f28157ebde2 100644 --- a/docs/en/engines/table-engines/special/memory.md +++ b/docs/en/engines/table-engines/special/memory.md @@ -45,6 +45,11 @@ Upper and lower bounds can be specified to limit Memory engine table size, effec CREATE TABLE memory (i UInt32) ENGINE = Memory SETTINGS min_rows_to_keep = 100, max_rows_to_keep = 1000; ``` +**Modify settings** +```sql +ALTER TABLE memory MODIFY SETTING min_rows_to_keep = 100, max_rows_to_keep = 1000; +``` + **Note:** Both `bytes` and `rows` capping parameters can be set at the same time, however, the lower bounds of `max` and `min` will be adhered to. ## Examples {#examples} @@ -97,3 +102,4 @@ SELECT total_bytes, total_rows FROM system.tables WHERE name = 'memory' and data │ 65536 │ 10000 │ └─────────────┴────────────┘ ``` + diff --git a/docs/en/getting-started/example-datasets/cell-towers.md b/docs/en/getting-started/example-datasets/cell-towers.md index 090de1b32fd..94fa6998f5d 100644 --- a/docs/en/getting-started/example-datasets/cell-towers.md +++ b/docs/en/getting-started/example-datasets/cell-towers.md @@ -75,14 +75,14 @@ This is the output of `DESCRIBE`. Down further in this guide the field type cho -1. Download the snapshot of the dataset from February 2021: [cell_towers.csv.xz](https://datasets.clickhouse.com/cell_towers.csv.xz) (729 MB). +1. Download the snapshot of the dataset from February 2021: [cell_towers.csv.xz](https://datasets.clickhouse.com/cell_towers.csv.xz) (686 MB). 2. Validate the integrity (optional step): ```bash md5sum cell_towers.csv.xz ``` ```response -8cf986f4a0d9f12c6f384a0e9192c908 cell_towers.csv.xz +8a797f7bdb55faba93f6cbc37d47b037 cell_towers.csv.xz ``` 3. Decompress it with the following command: @@ -132,7 +132,7 @@ SELECT radio, count() AS c FROM cell_towers GROUP BY radio ORDER BY c DESC ┌─radio─┬────────c─┐ │ UMTS │ 20686487 │ │ LTE │ 12101148 │ -│ GSM │ 9931312 │ +│ GSM │ 9931304 │ │ CDMA │ 556344 │ │ NR │ 867 │ └───────┴──────────┘ diff --git a/docs/en/getting-started/example-datasets/menus.md b/docs/en/getting-started/example-datasets/menus.md index 32fe62865d4..5a35c1d45bc 100644 --- a/docs/en/getting-started/example-datasets/menus.md +++ b/docs/en/getting-started/example-datasets/menus.md @@ -18,6 +18,9 @@ Run the command: ```bash wget https://s3.amazonaws.com/menusdata.nypl.org/gzips/2021_08_01_07_01_17_data.tgz +# Option: Validate the checksum +md5sum 2021_08_01_07_01_17_data.tgz +# Checksum should be equal to: db6126724de939a5481e3160a2d67d15 ``` Replace the link to the up to date link from http://menus.nypl.org/data if needed. diff --git a/docs/en/getting-started/example-datasets/opensky.md b/docs/en/getting-started/example-datasets/opensky.md index df28809495c..c0b4d96725d 100644 --- a/docs/en/getting-started/example-datasets/opensky.md +++ b/docs/en/getting-started/example-datasets/opensky.md @@ -7,7 +7,7 @@ title: "Crowdsourced air traffic data from The OpenSky Network 2020" The data in this dataset is derived and cleaned from the full OpenSky dataset to illustrate the development of air traffic during the COVID-19 pandemic. It spans all flights seen by the network's more than 2500 members since 1 January 2019. More data will be periodically included in the dataset until the end of the COVID-19 pandemic. -Source: https://zenodo.org/record/5092942#.YRBCyTpRXYd +Source: https://zenodo.org/records/5092942 Martin Strohmeier, Xavier Olive, Jannis Luebbe, Matthias Schaefer, and Vincent Lenders "Crowdsourced air traffic data from the OpenSky Network 2019–2020" @@ -19,7 +19,7 @@ https://doi.org/10.5194/essd-13-357-2021 Run the command: ```bash -wget -O- https://zenodo.org/record/5092942 | grep -oP 'https://zenodo.org/record/5092942/files/flightlist_\d+_\d+\.csv\.gz' | xargs wget +wget -O- https://zenodo.org/records/5092942 | grep -oE 'https://zenodo.org/records/5092942/files/flightlist_[0-9]+_[0-9]+\.csv\.gz' | xargs wget ``` Download will take about 2 minutes with good internet connection. There are 30 files with total size of 4.3 GB. @@ -127,15 +127,15 @@ Average flight distance is around 1000 km. Query: ```sql -SELECT avg(geoDistance(longitude_1, latitude_1, longitude_2, latitude_2)) FROM opensky; +SELECT round(avg(geoDistance(longitude_1, latitude_1, longitude_2, latitude_2)), 2) FROM opensky; ``` Result: ```text -┌─avg(geoDistance(longitude_1, latitude_1, longitude_2, latitude_2))─┐ -│ 1041090.6465708319 │ -└────────────────────────────────────────────────────────────────────┘ + ┌─round(avg(geoDistance(longitude_1, latitude_1, longitude_2, latitude_2)), 2)─┐ +1. │ 1041090.67 │ -- 1.04 million + └──────────────────────────────────────────────────────────────────────────────┘ ``` ### Most busy origin airports and the average distance seen {#busy-airports-average-distance} diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index ca689ef7995..e3be30bde8c 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -262,7 +262,7 @@ The required version can be downloaded with `curl` or `wget` from repository htt After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest stable version: ``` bash -LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ +LATEST_VERSION=$(curl -s https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/utils/list-versions/version_date.tsv | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index a76bb01ce9e..03cf345349e 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -79,7 +79,7 @@ The supported formats are: | [RowBinary](#rowbinary) | ✔ | ✔ | | [RowBinaryWithNames](#rowbinarywithnamesandtypes) | ✔ | ✔ | | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [RowBinaryWithDefaults](#rowbinarywithdefaults) | ✔ | ✔ | +| [RowBinaryWithDefaults](#rowbinarywithdefaults) | ✔ | ✗ | | [Native](#native) | ✔ | ✔ | | [Null](#null) | ✗ | ✔ | | [XML](#xml) | ✗ | ✔ | @@ -1270,12 +1270,13 @@ SELECT * FROM json_each_row_nested - [input_format_json_read_arrays_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_arrays_as_strings) - allow to parse JSON arrays as strings in JSON input formats. Default value - `true`. - [input_format_json_read_objects_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `true`. - [input_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings-formats.md/#input_format_json_named_tuples_as_objects) - parse named tuple columns as JSON objects. Default value - `true`. -- [input_format_json_try_infer_numbers_from_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_try_infer_numbers_from_strings) - Try to infer numbers from string fields while schema inference. Default value - `false`. +- [input_format_json_try_infer_numbers_from_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_try_infer_numbers_from_strings) - try to infer numbers from string fields while schema inference. Default value - `false`. - [input_format_json_try_infer_named_tuples_from_objects](/docs/en/operations/settings/settings-formats.md/#input_format_json_try_infer_named_tuples_from_objects) - try to infer named tuple from JSON objects during schema inference. Default value - `true`. - [input_format_json_infer_incomplete_types_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_infer_incomplete_types_as_strings) - use type String for keys that contains only Nulls or empty objects/arrays during schema inference in JSON input formats. Default value - `true`. - [input_format_json_defaults_for_missing_elements_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_defaults_for_missing_elements_in_named_tuple) - insert default values for missing elements in JSON object while parsing named tuple. Default value - `true`. -- [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - Ignore unknown keys in json object for named tuples. Default value - `false`. +- [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - ignore unknown keys in json object for named tuples. Default value - `false`. - [input_format_json_compact_allow_variable_number_of_columns](/docs/en/operations/settings/settings-formats.md/#input_format_json_compact_allow_variable_number_of_columns) - allow variable number of columns in JSONCompact/JSONCompactEachRow format, ignore extra columns and use default values on missing columns. Default value - `false`. +- [input_format_json_throw_on_bad_escape_sequence](/docs/en/operations/settings/settings-formats.md/#input_format_json_throw_on_bad_escape_sequence) - throw an exception if JSON string contains bad escape sequence. If disabled, bad escape sequences will remain as is in the data. Default value - `true`. - [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`. - [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`. - [output_format_json_quote_denormals](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`. @@ -1486,7 +1487,7 @@ Differs from [PrettySpaceNoEscapes](#prettyspacenoescapes) in that up to 10,000 - [output_format_pretty_max_value_width](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_max_value_width) - Maximum width of value to display in Pretty formats. If greater - it will be cut. Default value - `10000`. - [output_format_pretty_color](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_color) - use ANSI escape sequences to paint colors in Pretty formats. Default value - `true`. - [output_format_pretty_grid_charset](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_grid_charset) - Charset for printing grid borders. Available charsets: ASCII, UTF-8. Default value - `UTF-8`. -- [output_format_pretty_row_numbers](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_row_numbers) - Add row numbers before each row for pretty output format. Default value - `false`. +- [output_format_pretty_row_numbers](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_row_numbers) - Add row numbers before each row for pretty output format. Default value - `true`. ## RowBinary {#rowbinary} @@ -2356,7 +2357,7 @@ You can select data from a ClickHouse table and save them into some file in the $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filename.arrow} ``` -### Arrow format settings {#parquet-format-settings} +### Arrow format settings {#arrow-format-settings} - [output_format_arrow_low_cardinality_as_dictionary](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_low_cardinality_as_dictionary) - enable output ClickHouse LowCardinality type as Dictionary Arrow type. Default value - `false`. - [output_format_arrow_use_64_bit_indexes_for_dictionary](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_use_64_bit_indexes_for_dictionary) - use 64-bit integer type for Dictionary indexes. Default value - `false`. @@ -2464,7 +2465,7 @@ Result: ## Npy {#data-format-npy} -This function is designed to load a NumPy array from a .npy file into ClickHouse. The NumPy file format is a binary format used for efficiently storing arrays of numerical data. During import, ClickHouse treats top level dimension as an array of rows with single column. Supported Npy data types and their corresponding type in ClickHouse: +This function is designed to load a NumPy array from a .npy file into ClickHouse. The NumPy file format is a binary format used for efficiently storing arrays of numerical data. During import, ClickHouse treats top level dimension as an array of rows with single column. Supported Npy data types and their corresponding type in ClickHouse: | Npy type | ClickHouse type | |:--------:|:---------------:| | b1 | UInt8 | diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 4eeb19cefcf..bba5cde16f1 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -507,16 +507,18 @@ Example: ``` xml - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> GET TEST_HEADER_VALUE - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> predefined_query_handler - SELECT value FROM system.settings WHERE name = {name_1:String} - SELECT name, value FROM system.settings WHERE name = {name_2:String} + + SELECT name, value FROM system.settings + WHERE name IN ({name_1:String}, {name_2:String}) + @@ -524,13 +526,13 @@ Example: ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/query_param_with_url/1/max_threads/max_final_threads?max_threads=1&max_final_threads=2' -1 -max_final_threads 2 +$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_final_threads' 'http://localhost:8123/query_param_with_url/max_threads?max_threads=1&max_final_threads=2' +max_final_threads 2 +max_threads 1 ``` :::note -In one `predefined_query_handler` only supports one `query` of an insert type. +In one `predefined_query_handler` only one `query` is supported. ::: ### dynamic_query_handler {#dynamic_query_handler} diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index 8639af468c2..2ba50b39934 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -87,6 +87,7 @@ The BACKUP and RESTORE statements take a list of DATABASE and TABLE names, a des - `structure_only`: if enabled, allows to only backup or restore the CREATE statements without the data of tables - `storage_policy`: storage policy for the tables being restored. See [Using Multiple Block Devices for Data Storage](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes). This setting is only applicable to the `RESTORE` command. The specified storage policy applies only to tables with an engine from the `MergeTree` family. - `s3_storage_class`: the storage class used for S3 backup. For example, `STANDARD` + - `azure_attempt_to_create_container`: when using Azure Blob Storage, whether the specified container will try to be created if it doesn't exist. Default: true. ### Usage examples diff --git a/docs/en/operations/monitoring.md b/docs/en/operations/monitoring.md index de61da6f5c4..573e8075bca 100644 --- a/docs/en/operations/monitoring.md +++ b/docs/en/operations/monitoring.md @@ -3,6 +3,7 @@ slug: /en/operations/monitoring sidebar_position: 45 sidebar_label: Monitoring description: You can monitor the utilization of hardware resources and also ClickHouse server metrics. +keywords: [monitoring, observability, advanced dashboard, dashboard, observability dashboard] --- # Monitoring @@ -15,11 +16,11 @@ You can monitor: - Utilization of hardware resources. - ClickHouse server metrics. -## Built-in observability dashboard +## Built-in advanced observability dashboard Screenshot 2023-11-12 at 6 08 58 PM -ClickHouse comes with a built-in observability dashboard feature which can be accessed by `$HOST:$PORT/dashboard` (requires user and password) that shows the following metrics: +ClickHouse comes with a built-in advanced observability dashboard feature which can be accessed by `$HOST:$PORT/dashboard` (requires user and password) that shows the following metrics: - Queries/second - CPU usage (cores) - Queries running diff --git a/docs/en/operations/query-cache.md b/docs/en/operations/query-cache.md index a8532bc22b7..7a920671fc2 100644 --- a/docs/en/operations/query-cache.md +++ b/docs/en/operations/query-cache.md @@ -67,8 +67,7 @@ SETTINGS use_query_cache = true, enable_writes_to_query_cache = false; For maximum control, it is generally recommended to provide settings `use_query_cache`, `enable_writes_to_query_cache` and `enable_reads_from_query_cache` only with specific queries. It is also possible to enable caching at user or profile level (e.g. via `SET -use_query_cache = true`) but one should keep in mind that all `SELECT` queries including monitoring or debugging queries to system tables -may return cached results then. +use_query_cache = true`) but one should keep in mind that all `SELECT` queries may return cached results then. The query cache can be cleared using statement `SYSTEM DROP QUERY CACHE`. The content of the query cache is displayed in system table [system.query_cache](system-tables/query_cache.md). The number of query cache hits and misses since database start are shown as events @@ -175,6 +174,10 @@ Also, results of queries with non-deterministic functions are not cached by defa To force caching of results of queries with non-deterministic functions regardless, use setting [query_cache_nondeterministic_function_handling](settings/settings.md#query-cache-nondeterministic-function-handling). +Results of queries that involve system tables, e.g. `system.processes` or `information_schema.tables`, are not cached by default. To force +caching of results of queries with system tables regardless, use setting +[query_cache_system_table_handling](settings/settings.md#query-cache-system-table-handling). + :::note Prior to ClickHouse v23.11, setting 'query_cache_store_results_of_queries_with_nondeterministic_functions = 0 / 1' controlled whether results of queries with non-deterministic results were cached. In newer ClickHouse versions, this setting is obsolete and has no effect. diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 07c9a2b88ab..f87b6144deb 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -379,6 +379,18 @@ Type: UInt64 Default: 0 +## max_waiting_queries + +Limit on total number of concurrently waiting queries. Execution of a waiting query is blocked while required tables are loading asynchronously (see `async_load_databases`). Note that waiting queries are not counted when `max_concurrent_queries`, `max_concurrent_insert_queries`, `max_concurrent_select_queries`, `max_concurrent_queries_for_user` and `max_concurrent_queries_for_all_users` limits are checked. This correction is done to avoid hitting these limits just after server startup. Zero means unlimited. + +:::note +This setting can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged. +::: + +Type: UInt64 + +Default: 0 + ## max_connections Max server connections. @@ -424,7 +436,7 @@ Default: 0 Restriction on dropping partitions. If the size of a [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table exceeds `max_partition_size_to_drop` (in bytes), you can’t drop a partition using a [DROP PARTITION](../../sql-reference/statements/alter/partition.md#drop-partitionpart) query. -This setting does not require a restart of the Clickhouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. +This setting does not require a restart of the ClickHouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. Default value: 50 GB. The value 0 means that you can drop partitions without any restrictions. @@ -506,7 +518,7 @@ Restriction on deleting tables. If the size of a [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table exceeds `max_table_size_to_drop` (in bytes), you can’t delete it using a [DROP](../../sql-reference/statements/drop.md) query or [TRUNCATE](../../sql-reference/statements/truncate.md) query. -This setting does not require a restart of the Clickhouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. +This setting does not require a restart of the ClickHouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. Default value: 50 GB. The value 0 means that you can delete all tables without any restrictions. @@ -933,9 +945,9 @@ Hard limit is configured via system tools ## database_atomic_delay_before_drop_table_sec {#database_atomic_delay_before_drop_table_sec} -Sets the delay before remove table data in seconds. If the query has `SYNC` modifier, this setting is ignored. +The delay during which a dropped table can be restored using the [UNDROP](/docs/en/sql-reference/statements/undrop.md) statement. If `DROP TABLE` ran with a `SYNC` modifier, the setting is ignored. -Default value: `480` (8 minute). +Default value: `480` (8 minutes). ## database_catalog_unused_dir_hide_timeout_sec {#database_catalog_unused_dir_hide_timeout_sec} @@ -1342,6 +1354,7 @@ Keys: - `count` – The number of archived log files that ClickHouse stores. - `console` – Send `log` and `errorlog` to the console instead of file. To enable, set to `1` or `true`. - `stream_compress` – Compress `log` and `errorlog` with `lz4` stream compression. To enable, set to `1` or `true`. +- `formatting` – Specify log format to be printed in console log (currently only `json` supported). Both log and error log file names (only file names, not directories) support date and time format specifiers. @@ -1410,6 +1423,8 @@ Writing to the console can be configured. Config example: ``` +### syslog + Writing to the syslog is also supported. Config example: ``` xml @@ -1433,6 +1448,52 @@ Keys for syslog: Default value: `LOG_USER` if `address` is specified, `LOG_DAEMON` otherwise. - format – Message format. Possible values: `bsd` and `syslog.` +### Log formats + +You can specify the log format that will be outputted in the console log. Currently, only JSON is supported. Here is an example of an output JSON log: + +```json +{ + "date_time": "1650918987.180175", + "thread_name": "#1", + "thread_id": "254545", + "level": "Trace", + "query_id": "", + "logger_name": "BaseDaemon", + "message": "Received signal 2", + "source_file": "../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()", + "source_line": "192" +} +``` +To enable JSON logging support, use the following snippet: + +```xml + + + json + + date_time + thread_name + thread_id + level + query_id + logger_name + message + source_file + source_line + + + +``` + +**Renaming keys for JSON logs** + +Key names can be modified by changing tag values inside the `` tag. For example, to change `DATE_TIME` to `MY_DATE_TIME`, you can use `MY_DATE_TIME`. + +**Omitting keys for JSON logs** + +Log properties can be omitted by commenting out the property. For example, if you do not want your log to print `query_id`, you can comment out the `` tag. + ## send_crash_reports {#send_crash_reports} Settings for opt-in sending crash reports to the ClickHouse core developers team via [Sentry](https://sentry.io). @@ -1509,7 +1570,7 @@ Restriction on deleting tables. If the size of a [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table exceeds `max_table_size_to_drop` (in bytes), you can’t delete it using a [DROP](../../sql-reference/statements/drop.md) query or [TRUNCATE](../../sql-reference/statements/truncate.md) query. -This setting does not require a restart of the Clickhouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. +This setting does not require a restart of the ClickHouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. Default value: 50 GB. @@ -1527,7 +1588,7 @@ Restriction on dropping partitions. If the size of a [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table exceeds `max_partition_size_to_drop` (in bytes), you can’t drop a partition using a [DROP PARTITION](../../sql-reference/statements/alter/partition.md#drop-partitionpart) query. -This setting does not require a restart of the Clickhouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. +This setting does not require a restart of the ClickHouse server to apply. Another way to disable the restriction is to create the `/flags/force_drop_table` file. Default value: 50 GB. @@ -1725,7 +1786,7 @@ Default value: `0.5`. Asynchronous loading of databases and tables. -If `true` all non-system databases with `Ordinary`, `Atomic` and `Replicated` engine will be loaded asynchronously after the ClickHouse server start up. See `system.asynchronous_loader` table, `tables_loader_background_pool_size` and `tables_loader_foreground_pool_size` server settings. Any query that tries to access a table, that is not yet loaded, will wait for exactly this table to be started up. If load job fails, query will rethrow an error (instead of shutting down the whole server in case of `async_load_databases = false`). The table that is waited for by at least one query will be loaded with higher priority. DDL queries on a database will wait for exactly that database to be started up. +If `true` all non-system databases with `Ordinary`, `Atomic` and `Replicated` engine will be loaded asynchronously after the ClickHouse server start up. See `system.asynchronous_loader` table, `tables_loader_background_pool_size` and `tables_loader_foreground_pool_size` server settings. Any query that tries to access a table, that is not yet loaded, will wait for exactly this table to be started up. If load job fails, query will rethrow an error (instead of shutting down the whole server in case of `async_load_databases = false`). The table that is waited for by at least one query will be loaded with higher priority. DDL queries on a database will wait for exactly that database to be started up. Also consider setting a limit `max_waiting_queries` for the total number of waiting queries. If `false`, all databases are loaded when the server starts. @@ -2926,7 +2987,7 @@ Default: 0 ## ignore_empty_sql_security_in_create_view_query {#ignore_empty_sql_security_in_create_view_query} -If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. +If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. :::note This setting is only necessary for the migration period and will become obsolete in 24.4 diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index c7e461d15ae..9327d52227f 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -287,7 +287,7 @@ Default value: 0 (seconds) ## remote_fs_execute_merges_on_single_replica_time_threshold -When this setting has a value greater than than zero only a single replica starts the merge immediately if merged part on shared storage and `allow_remote_fs_zero_copy_replication` is enabled. +When this setting has a value greater than zero only a single replica starts the merge immediately if merged part on shared storage and `allow_remote_fs_zero_copy_replication` is enabled. :::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. @@ -867,3 +867,31 @@ Default value: `Never` Persists virtual column `_block_number` on merges. Default value: false. + +## exclude_deleted_rows_for_part_size_in_merge {#exclude_deleted_rows_for_part_size_in_merge} + +If enabled, estimated actual size of data parts (i.e., excluding those rows that have been deleted through `DELETE FROM`) will be used when selecting parts to merge. Note that this behavior is only triggered for data parts affected by `DELETE FROM` executed after this setting is enabled. + +Possible values: + +- true, false + +Default value: false + +**See Also** + +- [load_existing_rows_count_for_old_parts](#load_existing_rows_count_for_old_parts) setting + +## load_existing_rows_count_for_old_parts {#load_existing_rows_count_for_old_parts} + +If enabled along with [exclude_deleted_rows_for_part_size_in_merge](#exclude_deleted_rows_for_part_size_in_merge), deleted rows count for existing data parts will be calculated during table starting up. Note that it may slow down start up table loading. + +Possible values: + +- true, false + +Default value: false + +**See Also** + +- [exclude_deleted_rows_for_part_size_in_merge](#exclude_deleted_rows_for_part_size_in_merge) setting diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 477fbf94625..f455fcba840 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -651,6 +651,12 @@ This setting works only when setting `input_format_json_named_tuples_as_objects` Enabled by default. +## input_format_json_throw_on_bad_escape_sequence {#input_format_json_throw_on_bad_escape_sequence} + +Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data. + +Enabled by default. + ## output_format_json_array_of_rows {#output_format_json_array_of_rows} Enables the ability to output all rows as a JSON array in the [JSONEachRow](../../interfaces/formats.md/#jsoneachrow) format. @@ -1367,7 +1373,7 @@ Default value: `1'000'000`. While importing data, when column is not found in schema default value will be used instead of error. -Disabled by default. +Enabled by default. ### input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference {#input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference} @@ -1636,7 +1642,7 @@ Possible values: - 0 — Output without row numbers. - 1 — Output with row numbers. -Default value: `0`. +Default value: `1`. **Example** diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 6c427442e49..f9fe5f1b2d3 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1689,6 +1689,18 @@ Possible values: Default value: `throw`. +## query_cache_system_table_handling {#query-cache-system-table-handling} + +Controls how the [query cache](../query-cache.md) handles `SELECT` queries against system tables, i.e. tables in databases `system.*` and `information_schema.*`. + +Possible values: + +- `'throw'` - Throw an exception and don't cache the query result. +- `'save'` - Cache the query result. +- `'ignore'` - Don't cache the query result and don't throw an exception. + +Default value: `throw`. + ## query_cache_min_query_runs {#query-cache-min-query-runs} Minimum number of times a `SELECT` query must run before its result is stored in the [query cache](../query-cache.md). @@ -1776,7 +1788,7 @@ Default value: 0 (no restriction). ## insert_quorum {#insert_quorum} :::note -This setting is not applicable to SharedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information. +This setting is not applicable to SharedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information. ::: Enables the quorum writes. @@ -1819,7 +1831,7 @@ See also: ## insert_quorum_parallel {#insert_quorum_parallel} :::note -This setting is not applicable to SharedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information. +This setting is not applicable to SharedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information. ::: Enables or disables parallelism for quorum `INSERT` queries. If enabled, additional `INSERT` queries can be sent while previous queries have not yet finished. If disabled, additional writes to the same table will be rejected. @@ -1840,7 +1852,7 @@ See also: ## select_sequential_consistency {#select_sequential_consistency} :::note -This setting differ in behavior between SharedMergeTree and ReplicatedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information about the behavior of `select_sequential_consistency` in SharedMergeTree. +This setting differ in behavior between SharedMergeTree and ReplicatedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information about the behavior of `select_sequential_consistency` in SharedMergeTree. ::: Enables or disables sequential consistency for `SELECT` queries. Requires `insert_quorum_parallel` to be disabled (enabled by default). @@ -2817,6 +2829,17 @@ Possible values: Default value: 0. +## distributed_insert_skip_read_only_replicas {#distributed_insert_skip_read_only_replicas} + +Enables skipping read-only replicas for INSERT queries into Distributed. + +Possible values: + +- 0 — INSERT was as usual, if it will go to read-only replica it will fail +- 1 — Initiator will skip read-only replicas before sending data to shards. + +Default value: `0` + ## distributed_foreground_insert {#distributed_foreground_insert} Enables or disables synchronous data insertion into a [Distributed](../../engines/table-engines/special/distributed.md/#distributed) table. @@ -5291,7 +5314,7 @@ SETTINGS(dictionary_use_async_executor=1, max_threads=8); ## storage_metadata_write_full_object_key {#storage_metadata_write_full_object_key} When set to `true` the metadata files are written with `VERSION_FULL_OBJECT_KEY` format version. With that format full object storage key names are written to the metadata files. -When set to `false` the metadata files are written with the previous format version, `VERSION_INLINE_DATA`. With that format only suffixes of object storage key names are are written to the metadata files. The prefix for all of object storage key names is set in configurations files at `storage_configuration.disks` section. +When set to `false` the metadata files are written with the previous format version, `VERSION_INLINE_DATA`. With that format only suffixes of object storage key names are written to the metadata files. The prefix for all of object storage key names is set in configurations files at `storage_configuration.disks` section. Default value: `false`. @@ -5442,3 +5465,7 @@ Enabling this setting can lead to incorrect result as in case of evolved schema ::: Default value: 'false'. + +## allow_suspicious_primary_key {#allow_suspicious_primary_key} + +Allow suspicious `PRIMARY KEY`/`ORDER BY` for MergeTree (i.e. SimpleAggregateFunction). diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index fd81bc197d1..2c642dd2f0b 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -36,7 +36,7 @@ E.g. configuration option s3 https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -47,7 +47,7 @@ is equal to configuration (from `24.1`): s3 local https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -56,7 +56,7 @@ Configuration s3_plain https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -67,7 +67,7 @@ is equal to s3 plain https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -79,7 +79,7 @@ Example of full storage configuration will look like: s3 https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 @@ -105,7 +105,7 @@ Starting with 24.1 clickhouse version, it can also look like: s3 local https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 @@ -324,7 +324,7 @@ Configuration: s3_plain https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -337,7 +337,7 @@ Configuration: azure plain https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ - 1 + 1 ``` @@ -520,13 +520,13 @@ Example of configuration for versions later or equal to 22.8: - +
cache
- + ``` @@ -546,13 +546,13 @@ Example of configuration for versions earlier than 22.8: - +
s3
- + ``` diff --git a/docs/en/operations/system-tables/blob_storage_log.md b/docs/en/operations/system-tables/blob_storage_log.md index 2328f7f0346..8c0c33a504a 100644 --- a/docs/en/operations/system-tables/blob_storage_log.md +++ b/docs/en/operations/system-tables/blob_storage_log.md @@ -7,6 +7,7 @@ Contains logging entries with information about various blob storage operations Columns: +- `hostname` ([LowCardinality(String)](../../sql-reference/data-types/string.md)) — Hostname of the server executing the query. - `event_date` ([Date](../../sql-reference/data-types/date.md)) — Date of the event. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time of the event. - `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Time of the event with microseconds precision. @@ -38,6 +39,7 @@ SELECT * FROM system.blob_storage_log WHERE query_id = '7afe0450-504d-4e4b-9a80- ```text Row 1: ────── +hostname: clickhouse.eu-central1.internal event_date: 2023-10-31 event_time: 2023-10-31 16:03:40 event_time_microseconds: 2023-10-31 16:03:40.481437 diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index eaf79d035a9..d9800e05ff9 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -47,7 +47,7 @@ An example: ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024 --> 7500 - 1048576 + 1048576 8192 524288 false diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 3654cd157e9..38dca6b7071 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -483,7 +483,7 @@ Where: - `r1`- the number of unique visitors who visited the site during 2020-01-01 (the `cond1` condition). - `r2`- the number of unique visitors who visited the site during a specific time period between 2020-01-01 and 2020-01-02 (`cond1` and `cond2` conditions). -- `r3`- the number of unique visitors who visited the site during a specific time period between 2020-01-01 and 2020-01-03 (`cond1` and `cond3` conditions). +- `r3`- the number of unique visitors who visited the site during a specific time period on 2020-01-01 and 2020-01-03 (`cond1` and `cond3` conditions). ## uniqUpTo(N)(x) diff --git a/docs/en/sql-reference/aggregate-functions/reference/cramersv.md b/docs/en/sql-reference/aggregate-functions/reference/cramersv.md index e9e2c367610..2424ff95237 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/cramersv.md +++ b/docs/en/sql-reference/aggregate-functions/reference/cramersv.md @@ -7,26 +7,33 @@ sidebar_position: 351 [Cramer's V](https://en.wikipedia.org/wiki/Cram%C3%A9r%27s_V) (sometimes referred to as Cramer's phi) is a measure of association between two columns in a table. The result of the `cramersV` function ranges from 0 (corresponding to no association between the variables) to 1 and can reach 1 only when each value is completely determined by the other. It may be viewed as the association between two variables as a percentage of their maximum possible variation. +:::note +For a bias corrected version of Cramer's V see: [cramersVBiasCorrected](./cramersvbiascorrected.md) +::: + **Syntax** ``` sql cramersV(column1, column2) ``` -**Arguments** +**Parameters** -- `column1` and `column2` are the columns to be compared +- `column1`: first column to be compared. +- `column2`: second column to be compared. **Returned value** - a value between 0 (corresponding to no association between the columns' values) to 1 (complete association). -**Return type** is always [Float64](../../../sql-reference/data-types/float.md). +Type: always [Float64](../../../sql-reference/data-types/float.md). **Example** The following two columns being compared below have no association with each other, so the result of `cramersV` is 0: +Query: + ``` sql SELECT cramersV(a, b) diff --git a/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md b/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md index f5ad3a8a937..939c04e3fdc 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md +++ b/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md @@ -5,31 +5,31 @@ sidebar_position: 352 # cramersVBiasCorrected - Cramer's V is a measure of association between two columns in a table. The result of the [`cramersV` function](./cramersv.md) ranges from 0 (corresponding to no association between the variables) to 1 and can reach 1 only when each value is completely determined by the other. The function can be heavily biased, so this version of Cramer's V uses the [bias correction](https://en.wikipedia.org/wiki/Cram%C3%A9r%27s_V#Bias_correction). - - **Syntax** ``` sql cramersVBiasCorrected(column1, column2) ``` -**Arguments** +**Parameters** -- `column1` and `column2` are the columns to be compared +- `column1`: first column to be compared. +- `column2`: second column to be compared. **Returned value** - a value between 0 (corresponding to no association between the columns' values) to 1 (complete association). -**Return type** is always [Float64](../../../sql-reference/data-types/float.md). +Type: always [Float64](../../../sql-reference/data-types/float.md). **Example** The following two columns being compared below have a small association with each other. Notice the result of `cramersVBiasCorrected` is smaller than the result of `cramersV`: +Query: + ``` sql SELECT cramersV(a, b), diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md index 2f3efde859d..18f44d2fcc4 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md +++ b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md @@ -15,9 +15,9 @@ The `uniqCombined` function is a good choice for calculating the number of diffe **Arguments** -The function takes a variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types. +- `HLL_precision`: The base-2 logarithm of the number of cells in [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Optional, you can use the function as `uniqCombined(x[, ...])`. The default value for `HLL_precision` is 17, which is effectively 96 KiB of space (2^17 cells, 6 bits each). +- `X`: A variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types. -`HLL_precision` is the base-2 logarithm of the number of cells in [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Optional, you can use the function as `uniqCombined(x[, ...])`. The default value for `HLL_precision` is 17, which is effectively 96 KiB of space (2^17 cells, 6 bits each). **Returned value** @@ -25,26 +25,43 @@ The function takes a variable number of parameters. Parameters can be `Tuple`, ` **Implementation details** -Function: +The `uniqCombined` function: - Calculates a hash (64-bit hash for `String` and 32-bit otherwise) for all parameters in the aggregate, then uses it in calculations. - - Uses a combination of three algorithms: array, hash table, and HyperLogLog with an error correction table. - - For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. - + - For a small number of distinct elements, an array is used. + - When the set size is larger, a hash table is used. + - For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. - Provides the result deterministically (it does not depend on the query processing order). :::note -Since it uses 32-bit hash for non-`String` type, the result will have very high error for cardinalities significantly larger than `UINT_MAX` (error will raise quickly after a few tens of billions of distinct values), hence in this case you should use [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64) +Since it uses a 32-bit hash for non-`String` types, the result will have very high error for cardinalities significantly larger than `UINT_MAX` (error will raise quickly after a few tens of billions of distinct values), hence in this case you should use [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64). ::: -Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined`: +Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined` function: - Consumes several times less memory. - Calculates with several times higher accuracy. - Usually has slightly lower performance. In some scenarios, `uniqCombined` can perform better than `uniq`, for example, with distributed queries that transmit a large number of aggregation states over the network. +**Example** + +Query: + +```sql +SELECT uniqCombined(number) FROM numbers(1e6); +``` + +Result: + +```response +┌─uniqCombined(number)─┐ +│ 1001148 │ -- 1.00 million +└──────────────────────┘ +``` + +See the example section of [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64) for an example of the difference between `uniqCombined` and `uniqCombined64` for much larger inputs. + **See Also** - [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md index 9f010da57f2..b6e09bcaae3 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md +++ b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md @@ -5,4 +5,78 @@ sidebar_position: 193 # uniqCombined64 -Same as [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined), but uses 64-bit hash for all data types. +Calculates the approximate number of different argument values. It is the same as [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined), but uses a 64-bit hash for all data types rather than just for the String data type. + +``` sql +uniqCombined64(HLL_precision)(x[, ...]) +``` + +**Parameters** + +- `HLL_precision`: The base-2 logarithm of the number of cells in [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Optionally, you can use the function as `uniqCombined64(x[, ...])`. The default value for `HLL_precision` is 17, which is effectively 96 KiB of space (2^17 cells, 6 bits each). +- `X`: A variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types. + +**Returned value** + +- A number [UInt64](../../../sql-reference/data-types/int-uint.md)-type number. + +**Implementation details** + +The `uniqCombined64` function: +- Calculates a hash (64-bit hash for all data types) for all parameters in the aggregate, then uses it in calculations. +- Uses a combination of three algorithms: array, hash table, and HyperLogLog with an error correction table. + - For a small number of distinct elements, an array is used. + - When the set size is larger, a hash table is used. + - For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. +- Provides the result deterministically (it does not depend on the query processing order). + +:::note +Since it uses 64-bit hash for all types, the result does not suffer from very high error for cardinalities significantly larger than `UINT_MAX` like [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md) does, which uses a 32-bit hash for non-`String` types. +::: + +Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined64` function: + +- Consumes several times less memory. +- Calculates with several times higher accuracy. + +**Example** + +In the example below `uniqCombined64` is run on `1e10` different numbers returning a very close approximation of the number of different argument values. + +Query: + +```sql +SELECT uniqCombined64(number) FROM numbers(1e10); +``` + +Result: + +```response +┌─uniqCombined64(number)─┐ +│ 9998568925 │ -- 10.00 billion +└────────────────────────┘ +``` + +By comparison the `uniqCombined` function returns a rather poor approximation for an input this size. + +Query: + +```sql +SELECT uniqCombined(number) FROM numbers(1e10); +``` + +Result: + +```response +┌─uniqCombined(number)─┐ +│ 5545308725 │ -- 5.55 billion +└──────────────────────┘ +``` + +**See Also** + +- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) +- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md) +- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12) +- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact) +- [uniqTheta](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch) diff --git a/docs/en/sql-reference/data-types/aggregatefunction.md b/docs/en/sql-reference/data-types/aggregatefunction.md index fe6d7ebe0dc..87511a505dc 100644 --- a/docs/en/sql-reference/data-types/aggregatefunction.md +++ b/docs/en/sql-reference/data-types/aggregatefunction.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/aggregatefunction -sidebar_position: 53 +sidebar_position: 46 sidebar_label: AggregateFunction --- diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index 0ee7c8de93c..e5a8ce5d18b 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/array -sidebar_position: 52 +sidebar_position: 32 sidebar_label: Array(T) --- diff --git a/docs/en/sql-reference/data-types/boolean.md b/docs/en/sql-reference/data-types/boolean.md index 70abf767a41..4c59bd947de 100644 --- a/docs/en/sql-reference/data-types/boolean.md +++ b/docs/en/sql-reference/data-types/boolean.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/boolean -sidebar_position: 43 +sidebar_position: 22 sidebar_label: Boolean --- diff --git a/docs/en/sql-reference/data-types/date.md b/docs/en/sql-reference/data-types/date.md index 26e4610aec7..7adee3bbf3c 100644 --- a/docs/en/sql-reference/data-types/date.md +++ b/docs/en/sql-reference/data-types/date.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/date -sidebar_position: 47 +sidebar_position: 12 sidebar_label: Date --- diff --git a/docs/en/sql-reference/data-types/date32.md b/docs/en/sql-reference/data-types/date32.md index 38a07cd817d..a08c931b7fc 100644 --- a/docs/en/sql-reference/data-types/date32.md +++ b/docs/en/sql-reference/data-types/date32.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/date32 -sidebar_position: 48 +sidebar_position: 14 sidebar_label: Date32 --- diff --git a/docs/en/sql-reference/data-types/datetime.md b/docs/en/sql-reference/data-types/datetime.md index 1adff18f598..ac9a72c2641 100644 --- a/docs/en/sql-reference/data-types/datetime.md +++ b/docs/en/sql-reference/data-types/datetime.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/datetime -sidebar_position: 48 +sidebar_position: 16 sidebar_label: DateTime --- @@ -36,9 +36,9 @@ You can explicitly set a time zone for `DateTime`-type columns when creating a t The [clickhouse-client](../../interfaces/cli.md) applies the server time zone by default if a time zone isn’t explicitly set when initializing the data type. To use the client time zone, run `clickhouse-client` with the `--use_client_time_zone` parameter. -ClickHouse outputs values depending on the value of the [date_time_output_format](../../operations/settings/settings.md#settings-date_time_output_format) setting. `YYYY-MM-DD hh:mm:ss` text format by default. Additionally, you can change the output with the [formatDateTime](../../sql-reference/functions/date-time-functions.md#formatdatetime) function. +ClickHouse outputs values depending on the value of the [date_time_output_format](../../operations/settings/settings-formats.md#date_time_output_format) setting. `YYYY-MM-DD hh:mm:ss` text format by default. Additionally, you can change the output with the [formatDateTime](../../sql-reference/functions/date-time-functions.md#formatdatetime) function. -When inserting data into ClickHouse, you can use different formats of date and time strings, depending on the value of the [date_time_input_format](../../operations/settings/settings.md#settings-date_time_input_format) setting. +When inserting data into ClickHouse, you can use different formats of date and time strings, depending on the value of the [date_time_input_format](../../operations/settings/settings-formats.md#date_time_input_format) setting. ## Examples @@ -147,8 +147,8 @@ Time shifts for multiple days. Some pacific islands changed their timezone offse - [Type conversion functions](../../sql-reference/functions/type-conversion-functions.md) - [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md) - [Functions for working with arrays](../../sql-reference/functions/array-functions.md) -- [The `date_time_input_format` setting](../../operations/settings/settings-formats.md#settings-date_time_input_format) -- [The `date_time_output_format` setting](../../operations/settings/settings-formats.md#settings-date_time_output_format) +- [The `date_time_input_format` setting](../../operations/settings/settings-formats.md#date_time_input_format) +- [The `date_time_output_format` setting](../../operations/settings/settings-formats.md#date_time_output_format) - [The `timezone` server configuration parameter](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) - [The `session_timezone` setting](../../operations/settings/settings.md#session_timezone) - [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime) diff --git a/docs/en/sql-reference/data-types/datetime64.md b/docs/en/sql-reference/data-types/datetime64.md index 504d0e2b0a6..ef452a723e6 100644 --- a/docs/en/sql-reference/data-types/datetime64.md +++ b/docs/en/sql-reference/data-types/datetime64.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/datetime64 -sidebar_position: 49 +sidebar_position: 18 sidebar_label: DateTime64 --- diff --git a/docs/en/sql-reference/data-types/decimal.md b/docs/en/sql-reference/data-types/decimal.md index 2b32e72a28f..dfdefdff5a5 100644 --- a/docs/en/sql-reference/data-types/decimal.md +++ b/docs/en/sql-reference/data-types/decimal.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/decimal -sidebar_position: 42 +sidebar_position: 6 sidebar_label: Decimal --- diff --git a/docs/en/sql-reference/data-types/enum.md b/docs/en/sql-reference/data-types/enum.md index 02e73a0360e..ccfeb7f3416 100644 --- a/docs/en/sql-reference/data-types/enum.md +++ b/docs/en/sql-reference/data-types/enum.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/enum -sidebar_position: 50 +sidebar_position: 20 sidebar_label: Enum --- diff --git a/docs/en/sql-reference/data-types/fixedstring.md b/docs/en/sql-reference/data-types/fixedstring.md index a56b3fccbc1..0316df7fe34 100644 --- a/docs/en/sql-reference/data-types/fixedstring.md +++ b/docs/en/sql-reference/data-types/fixedstring.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/fixedstring -sidebar_position: 45 +sidebar_position: 10 sidebar_label: FixedString(N) --- -# FixedString +# FixedString(N) A fixed-length string of `N` bytes (neither characters nor code points). diff --git a/docs/en/sql-reference/data-types/float.md b/docs/en/sql-reference/data-types/float.md index be7b2a7fcd8..23131d5b4fe 100644 --- a/docs/en/sql-reference/data-types/float.md +++ b/docs/en/sql-reference/data-types/float.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/float -sidebar_position: 41 +sidebar_position: 4 sidebar_label: Float32, Float64 --- diff --git a/docs/en/sql-reference/data-types/geo.md b/docs/en/sql-reference/data-types/geo.md index 1d37b829dd5..7e3c32b3451 100644 --- a/docs/en/sql-reference/data-types/geo.md +++ b/docs/en/sql-reference/data-types/geo.md @@ -1,8 +1,8 @@ --- slug: /en/sql-reference/data-types/geo -sidebar_position: 62 +sidebar_position: 54 sidebar_label: Geo -title: "Geo Data Types" +title: "Geometric" --- ClickHouse supports data types for representing geographical objects — locations, lands, etc. diff --git a/docs/en/sql-reference/data-types/index.md b/docs/en/sql-reference/data-types/index.md index ffd063590fa..fcb0b60d022 100644 --- a/docs/en/sql-reference/data-types/index.md +++ b/docs/en/sql-reference/data-types/index.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/ sidebar_label: List of data types -sidebar_position: 37 +sidebar_position: 1 --- -# ClickHouse Data Types +# Data Types in ClickHouse ClickHouse can store various kinds of data in table cells. This section describes the supported data types and special considerations for using and/or implementing them if any. diff --git a/docs/en/sql-reference/data-types/int-uint.md b/docs/en/sql-reference/data-types/int-uint.md index 520454a859f..52d2982de19 100644 --- a/docs/en/sql-reference/data-types/int-uint.md +++ b/docs/en/sql-reference/data-types/int-uint.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/int-uint -sidebar_position: 40 +sidebar_position: 2 sidebar_label: UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 --- diff --git a/docs/en/sql-reference/data-types/ipv4.md b/docs/en/sql-reference/data-types/ipv4.md index 288806f47b3..637ed543e08 100644 --- a/docs/en/sql-reference/data-types/ipv4.md +++ b/docs/en/sql-reference/data-types/ipv4.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/ipv4 -sidebar_position: 59 +sidebar_position: 28 sidebar_label: IPv4 --- diff --git a/docs/en/sql-reference/data-types/ipv6.md b/docs/en/sql-reference/data-types/ipv6.md index 97959308b58..642a7db81fc 100644 --- a/docs/en/sql-reference/data-types/ipv6.md +++ b/docs/en/sql-reference/data-types/ipv6.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/ipv6 -sidebar_position: 60 +sidebar_position: 30 sidebar_label: IPv6 --- diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md index fd548a0d5a2..39e37abad82 100644 --- a/docs/en/sql-reference/data-types/json.md +++ b/docs/en/sql-reference/data-types/json.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/json -sidebar_position: 54 +sidebar_position: 26 sidebar_label: JSON --- diff --git a/docs/en/sql-reference/data-types/lowcardinality.md b/docs/en/sql-reference/data-types/lowcardinality.md index db10103282d..133ac2bd72e 100644 --- a/docs/en/sql-reference/data-types/lowcardinality.md +++ b/docs/en/sql-reference/data-types/lowcardinality.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/lowcardinality -sidebar_position: 51 -sidebar_label: LowCardinality +sidebar_position: 42 +sidebar_label: LowCardinality(T) --- -# LowCardinality +# LowCardinality(T) Changes the internal representation of other data types to be dictionary-encoded. diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md index e0c8b98f9f8..2c734969afc 100644 --- a/docs/en/sql-reference/data-types/map.md +++ b/docs/en/sql-reference/data-types/map.md @@ -1,12 +1,12 @@ --- slug: /en/sql-reference/data-types/map -sidebar_position: 65 -sidebar_label: Map(key, value) +sidebar_position: 36 +sidebar_label: Map(K, V) --- -# Map(key, value) +# Map(K, V) -`Map(key, value)` data type stores `key:value` pairs. +`Map(K, V)` data type stores `key:value` pairs. **Parameters** diff --git a/docs/en/sql-reference/data-types/multiword-types.md b/docs/en/sql-reference/data-types/multiword-types.md deleted file mode 100644 index ebbe1d84544..00000000000 --- a/docs/en/sql-reference/data-types/multiword-types.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -slug: /en/sql-reference/data-types/multiword-types -sidebar_position: 61 -sidebar_label: Multiword Type Names -title: "Multiword Types" ---- - -When creating tables, you can use data types with a name consisting of several words. This is implemented for better SQL compatibility. - -## Multiword Types Support - -| Multiword types | Simple types | -|----------------------------------|--------------------------------------------------------------| -| DOUBLE PRECISION | [Float64](../../sql-reference/data-types/float.md) | -| CHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| CHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | -| NCHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| NCHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHAR | [String](../../sql-reference/data-types/string.md) | -| BINARY LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| BINARY VARYING | [String](../../sql-reference/data-types/string.md) | diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 5504765e4a0..abcb87a0c1b 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -1,7 +1,7 @@ --- slug: /en/sql-reference/data-types/nullable -sidebar_position: 55 -sidebar_label: Nullable +sidebar_position: 44 +sidebar_label: Nullable(T) --- # Nullable(T) diff --git a/docs/en/sql-reference/data-types/simpleaggregatefunction.md b/docs/en/sql-reference/data-types/simpleaggregatefunction.md index 517a28576f0..39f8409c1e1 100644 --- a/docs/en/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/en/sql-reference/data-types/simpleaggregatefunction.md @@ -1,5 +1,7 @@ --- slug: /en/sql-reference/data-types/simpleaggregatefunction +sidebar_position: 48 +sidebar_label: SimpleAggregateFunction --- # SimpleAggregateFunction diff --git a/docs/en/sql-reference/data-types/string.md b/docs/en/sql-reference/data-types/string.md index f891a9303e5..8a4f346fdfc 100644 --- a/docs/en/sql-reference/data-types/string.md +++ b/docs/en/sql-reference/data-types/string.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/string -sidebar_position: 44 +sidebar_position: 8 sidebar_label: String --- @@ -13,7 +13,7 @@ When creating tables, numeric parameters for string fields can be set (e.g. `VAR Aliases: -- `String` — `LONGTEXT`, `MEDIUMTEXT`, `TINYTEXT`, `TEXT`, `LONGBLOB`, `MEDIUMBLOB`, `TINYBLOB`, `BLOB`, `VARCHAR`, `CHAR`. +- `String` — `LONGTEXT`, `MEDIUMTEXT`, `TINYTEXT`, `TEXT`, `LONGBLOB`, `MEDIUMBLOB`, `TINYBLOB`, `BLOB`, `VARCHAR`, `CHAR`, `CHAR LARGE OBJECT`, `CHAR VARYING`, `CHARACTER LARGE OBJECT`, `CHARACTER VARYING`, `NCHAR LARGE OBJECT`, `NCHAR VARYING`, `NATIONAL CHARACTER LARGE OBJECT`, `NATIONAL CHARACTER VARYING`, `NATIONAL CHAR VARYING`, `NATIONAL CHARACTER`, `NATIONAL CHAR`, `BINARY LARGE OBJECT`, `BINARY VARYING`, ## Encodings diff --git a/docs/en/sql-reference/data-types/tuple.md b/docs/en/sql-reference/data-types/tuple.md index 8f87eeca075..0525a3b0476 100644 --- a/docs/en/sql-reference/data-types/tuple.md +++ b/docs/en/sql-reference/data-types/tuple.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/tuple -sidebar_position: 54 +sidebar_position: 34 sidebar_label: Tuple(T1, T2, ...) --- -# Tuple(T1, T2, …) +# Tuple(T1, T2, ...) A tuple of elements, each having an individual [type](../../sql-reference/data-types/index.md#data_types). Tuple must contain at least one element. diff --git a/docs/en/sql-reference/data-types/uuid.md b/docs/en/sql-reference/data-types/uuid.md index 40f756b9588..75e163f5063 100644 --- a/docs/en/sql-reference/data-types/uuid.md +++ b/docs/en/sql-reference/data-types/uuid.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/data-types/uuid -sidebar_position: 46 +sidebar_position: 24 sidebar_label: UUID --- diff --git a/docs/en/sql-reference/data-types/variant.md b/docs/en/sql-reference/data-types/variant.md index 7d10d4b0e97..3c2b6e0a362 100644 --- a/docs/en/sql-reference/data-types/variant.md +++ b/docs/en/sql-reference/data-types/variant.md @@ -1,10 +1,10 @@ --- slug: /en/sql-reference/data-types/variant -sidebar_position: 55 -sidebar_label: Variant +sidebar_position: 40 +sidebar_label: Variant(T1, T2, ...) --- -# Variant(T1, T2, T3, ...) +# Variant(T1, T2, ...) This type represents a union of other data types. Type `Variant(T1, T2, ..., TN)` means that each row of this type has a value of either type `T1` or `T2` or ... or `TN` or none of them (`NULL` value). @@ -190,22 +190,67 @@ SELECT toTypeName(variantType(v)) FROM test LIMIT 1; └─────────────────────────────────────────────────────────────────────┘ ``` -## Conversion between Variant column and other columns +## Conversion between a Variant column and other columns -There are 3 possible conversions that can be performed with Variant column. +There are 4 possible conversions that can be performed with a column of type `Variant`. -### Converting an ordinary column to a Variant column +### Converting a String column to a Variant column -It is possible to convert ordinary column with type `T` to a `Variant` column containing this type: +Conversion from `String` to `Variant` is performed by parsing a value of `Variant` type from the string value: ```sql -SELECT toTypeName(variant) as type_name, 'Hello, World!'::Variant(UInt64, String, Array(UInt64)) as variant; +SELECT '42'::Variant(String, UInt64) as variant, variantType(variant) as variant_type ``` ```text -┌─type_name──────────────────────────────┬─variant───────┐ -│ Variant(Array(UInt64), String, UInt64) │ Hello, World! │ -└────────────────────────────────────────┴───────────────┘ +┌─variant─┬─variant_type─┐ +│ 42 │ UInt64 │ +└─────────┴──────────────┘ +``` + +```sql +SELECT '[1, 2, 3]'::Variant(String, Array(UInt64)) as variant, variantType(variant) as variant_type +``` + +```text +┌─variant─┬─variant_type──┐ +│ [1,2,3] │ Array(UInt64) │ +└─────────┴───────────────┘ +``` + +```sql +SELECT CAST(map('key1', '42', 'key2', 'true', 'key3', '2020-01-01'), 'Map(String, Variant(UInt64, Bool, Date))') as map_of_variants, mapApply((k, v) -> (k, variantType(v)), map_of_variants) as map_of_variant_types``` +``` + +```text +┌─map_of_variants─────────────────────────────┬─map_of_variant_types──────────────────────────┐ +│ {'key1':42,'key2':true,'key3':'2020-01-01'} │ {'key1':'UInt64','key2':'Bool','key3':'Date'} │ +└─────────────────────────────────────────────┴───────────────────────────────────────────────┘ +``` + +### Converting an ordinary column to a Variant column + +It is possible to convert an ordinary column with type `T` to a `Variant` column containing this type: + +```sql +SELECT toTypeName(variant) as type_name, [1,2,3]::Array(UInt64)::Variant(UInt64, String, Array(UInt64)) as variant, variantType(variant) as variant_name + ``` + +```text +┌─type_name──────────────────────────────┬─variant─┬─variant_name──┐ +│ Variant(Array(UInt64), String, UInt64) │ [1,2,3] │ Array(UInt64) │ +└────────────────────────────────────────┴─────────┴───────────────┘ +``` + +Note: converting from `String` type is always performed through parsing, if you need to convert `String` column to `String` variant of a `Variant` without parsing, you can do the following: +```sql +SELECT '[1, 2, 3]'::Variant(String)::Variant(String, Array(UInt64), UInt64) as variant, variantType(variant) as variant_type +``` + +```sql +┌─variant───┬─variant_type─┐ +│ [1, 2, 3] │ String │ +└───────────┴──────────────┘ ``` ### Converting a Variant column to an ordinary column @@ -395,3 +440,37 @@ SELECT v, variantType(v) FROM test ORDER by v; │ 100 │ UInt32 │ └─────┴────────────────┘ ``` + +## JSONExtract functions with Variant + +All `JSONExtract*` functions support `Variant` type: + +```sql +SELECT JSONExtract('{"a" : [1, 2, 3]}', 'a', 'Variant(UInt32, String, Array(UInt32))') AS variant, variantType(variant) AS variant_type; +``` + +```text +┌─variant─┬─variant_type──┐ +│ [1,2,3] │ Array(UInt32) │ +└─────────┴───────────────┘ +``` + +```sql +SELECT JSONExtract('{"obj" : {"a" : 42, "b" : "Hello", "c" : [1,2,3]}}', 'obj', 'Map(String, Variant(UInt32, String, Array(UInt32)))') AS map_of_variants, mapApply((k, v) -> (k, variantType(v)), map_of_variants) AS map_of_variant_types +``` + +```text +┌─map_of_variants──────────────────┬─map_of_variant_types────────────────────────────┐ +│ {'a':42,'b':'Hello','c':[1,2,3]} │ {'a':'UInt32','b':'String','c':'Array(UInt32)'} │ +└──────────────────────────────────┴─────────────────────────────────────────────────┘ +``` + +```sql +SELECT JSONExtractKeysAndValues('{"a" : 42, "b" : "Hello", "c" : [1,2,3]}', 'Variant(UInt32, String, Array(UInt32))') AS variants, arrayMap(x -> (x.1, variantType(x.2)), variants) AS variant_types +``` + +```text +┌─variants───────────────────────────────┬─variant_types─────────────────────────────────────────┐ +│ [('a',42),('b','Hello'),('c',[1,2,3])] │ [('a','UInt32'),('b','String'),('c','Array(UInt32)')] │ +└────────────────────────────────────────┴───────────────────────────────────────────────────────┘ +``` diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index ba7695af3fa..3bb9d4e7dbe 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1670,7 +1670,7 @@ Like [fromDaysSinceYearZero](#fromDaysSinceYearZero) but returns a [Date32](../. ## age -Returns the `unit` component of the difference between `startdate` and `enddate`. The difference is calculated using a precision of 1 microsecond. +Returns the `unit` component of the difference between `startdate` and `enddate`. The difference is calculated using a precision of 1 nanosecond. E.g. the difference between `2021-12-29` and `2022-01-01` is 3 days for `day` unit, 0 months for `month` unit, 0 years for `year` unit. For an alternative to `age`, see function `date\_diff`. @@ -1686,16 +1686,17 @@ age('unit', startdate, enddate, [timezone]) - `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md). Possible values: - - `microsecond` `microseconds` `us` `u` - - `millisecond` `milliseconds` `ms` - - `second` `seconds` `ss` `s` - - `minute` `minutes` `mi` `n` - - `hour` `hours` `hh` `h` - - `day` `days` `dd` `d` - - `week` `weeks` `wk` `ww` - - `month` `months` `mm` `m` - - `quarter` `quarters` `qq` `q` - - `year` `years` `yyyy` `yy` + - `nanosecond`, `nanoseconds`, `ns` + - `microsecond`, `microseconds`, `us`, `u` + - `millisecond`, `milliseconds`, `ms` + - `second`, `seconds`, `ss`, `s` + - `minute`, `minutes`, `mi`, `n` + - `hour`, `hours`, `hh`, `h` + - `day`, `days`, `dd`, `d` + - `week`, `weeks`, `wk`, `ww` + - `month`, `months`, `mm`, `m` + - `quarter`, `quarters`, `qq`, `q` + - `year`, `years`, `yyyy`, `yy` - `startdate` — The first time value to subtract (the subtrahend). [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). @@ -1763,16 +1764,17 @@ Aliases: `dateDiff`, `DATE_DIFF`, `timestampDiff`, `timestamp_diff`, `TIMESTAMP_ - `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md). Possible values: - - `microsecond` `microseconds` `us` `u` - - `millisecond` `milliseconds` `ms` - - `second` `seconds` `ss` `s` - - `minute` `minutes` `mi` `n` - - `hour` `hours` `hh` `h` - - `day` `days` `dd` `d` - - `week` `weeks` `wk` `ww` - - `month` `months` `mm` `m` - - `quarter` `quarters` `qq` `q` - - `year` `years` `yyyy` `yy` + - `nanosecond`, `nanoseconds`, `ns` + - `microsecond`, `microseconds`, `us`, `u` + - `millisecond`, `milliseconds`, `ms` + - `second`, `seconds`, `ss`, `s` + - `minute`, `minutes`, `mi`, `n` + - `hour`, `hours`, `hh`, `h` + - `day`, `days`, `dd`, `d` + - `week`, `weeks`, `wk`, `ww` + - `month`, `months`, `mm`, `m` + - `quarter`, `quarters`, `qq`, `q` + - `year`, `years`, `yyyy`, `yy` - `startdate` — The first time value to subtract (the subtrahend). [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). @@ -1904,7 +1906,7 @@ Aliases: `dateAdd`, `DATE_ADD`. **Arguments** -- `unit` — The type of interval to add. [String](../../sql-reference/data-types/string.md). +- `unit` — The type of interval to add. Note: This is not a [String](../../sql-reference/data-types/string.md) and must therefore not be quoted. Possible values: - `second` @@ -1959,7 +1961,7 @@ Aliases: `dateSub`, `DATE_SUB`. **Arguments** -- `unit` — The type of interval to subtract. Note: The unit should be unquoted. +- `unit` — The type of interval to subtract. Note: This is not a [String](../../sql-reference/data-types/string.md) and must therefore not be quoted. Possible values: diff --git a/docs/en/sql-reference/functions/distance-functions.md b/docs/en/sql-reference/functions/distance-functions.md index e20c35c6b6f..5f3514049c7 100644 --- a/docs/en/sql-reference/functions/distance-functions.md +++ b/docs/en/sql-reference/functions/distance-functions.md @@ -81,6 +81,43 @@ Result: │ 2.23606797749979 │ └──────────────────┘ ``` +## L2SquaredNorm + +Calculates the square root of the sum of the squares of the vector values (the [L2Norm](#l2norm)) squared. + +**Syntax** + +```sql +L2SquaredNorm(vector) +``` + +Alias: `normL2Squared`. + +***Arguments** + +- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- L2-norm squared. + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT L2SquaredNorm((1, 2)); +``` + +Result: + +```text +┌─L2SquaredNorm((1, 2))─┐ +│ 5 │ +└───────────────────────┘ +``` ## LinfNorm diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 90c7d8c2206..1cd7eeb7c83 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -594,6 +594,45 @@ Calculates JumpConsistentHash form a UInt64. Accepts two arguments: a UInt64-type key and the number of buckets. Returns Int32. For more information, see the link: [JumpConsistentHash](https://arxiv.org/pdf/1406.2294.pdf) +## kostikConsistentHash + +An O(1) time and space consistent hash algorithm by Konstantin 'kostik' Oblakov. Previously `yandexConsistentHash`. + +**Syntax** + +```sql +kostikConsistentHash(input, n) +``` + +Alias: `yandexConsistentHash` (left for backwards compatibility sake). + +**Parameters** + +- `input`: A UInt64-type key [UInt64](/docs/en/sql-reference/data-types/int-uint.md). +- `n`: Number of buckets. [UInt16](/docs/en/sql-reference/data-types/int-uint.md). + +**Returned value** + +- A [UInt16](/docs/en/sql-reference/data-types/int-uint.md) data type hash value. + +**Implementation details** + +It is efficient only if n <= 32768. + +**Example** + +Query: + +```sql +SELECT kostikConsistentHash(16045690984833335023, 2); +``` + +```response +┌─kostikConsistentHash(16045690984833335023, 2)─┐ +│ 1 │ +└───────────────────────────────────────────────┘ +``` + ## murmurHash2_32, murmurHash2_64 Produces a [MurmurHash2](https://github.com/aappleby/smhasher) hash value. @@ -1153,6 +1192,42 @@ Result: └────────────┘ ``` +## wyHash64 + +Produces a 64-bit [wyHash64](https://github.com/wangyi-fudan/wyhash) hash value. + +**Syntax** + +```sql +wyHash64(string) +``` + +**Arguments** + +- `string` — String. [String](/docs/en/sql-reference/data-types/string.md). + +**Returned value** + +- Hash value. + +Type: [UInt64](/docs/en/sql-reference/data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT wyHash64('ClickHouse') AS Hash; +``` + +Result: + +```response +┌─────────────────Hash─┐ +│ 12336419557878201794 │ +└──────────────────────┘ +``` + ## ngramMinHash Splits a ASCII string into n-grams of `ngramsize` symbols and calculates hash values for each n-gram. Uses `hashnum` minimum hashes to calculate the minimum hash and `hashnum` maximum hashes to calculate the maximum hash. Returns a tuple with these hashes. Is case sensitive. diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index e7fca31483a..187f248e92d 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -405,7 +405,7 @@ Returns the name of the current user. In case of a distributed query, the name o SELECT currentUser(); ``` -Alias: `user()`, `USER()`. +Aliases: `user()`, `USER()`, `current_user()`. Aliases are case insensitive. **Returned values** @@ -543,12 +543,64 @@ You can get similar result by using the [ternary operator](../../sql-reference/f Returns 1 if the Float32 and Float64 argument is NaN, otherwise this function 0. -## hasColumnInTable(\[‘hostname’\[, ‘username’\[, ‘password’\]\],\] ‘database’, ‘table’, ‘column’) +## hasColumnInTable + +Given the database name, the table name, and the column name as constant strings, returns 1 if the given column exists, otherwise 0. + +**Syntax** + +```sql +hasColumnInTable(\[‘hostname’\[, ‘username’\[, ‘password’\]\],\] ‘database’, ‘table’, ‘column’) +``` + +**Parameters** + +- `database` : name of the database. [String literal](../syntax#syntax-string-literal) +- `table` : name of the table. [String literal](../syntax#syntax-string-literal) +- `column` : name of the column. [String literal](../syntax#syntax-string-literal) +- `hostname` : remote server name to perform the check on. [String literal](../syntax#syntax-string-literal) +- `username` : username for remote server. [String literal](../syntax#syntax-string-literal) +- `password` : password for remote server. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- `1` if the given column exists. +- `0`, otherwise. + +**Implementation details** -Given the database name, the table name, and the column name as constant strings, returns 1 if the given column exists, otherwise 0. If parameter `hostname` is given, the check is performed on a remote server. -If the table does not exist, an exception is thrown. For elements in a nested data structure, the function checks for the existence of a column. For the nested data structure itself, the function returns 0. +**Example** + +Query: + +```sql +SELECT hasColumnInTable('system','metrics','metric') +``` + +```response +1 +``` + +```sql +SELECT hasColumnInTable('system','metrics','non-existing_column') +``` + +```response +0 +``` + +## hasThreadFuzzer + +Returns whether Thread Fuzzer is effective. It can be used in tests to prevent runs from being too long. + +**Syntax** + +```sql +hasThreadFuzzer(); +``` + ## bar Builds a bar chart. @@ -864,6 +916,34 @@ Returns the larger value of a and b. Returns the server’s uptime in seconds. If executed in the context of a distributed table, this function generates a normal column with values relevant to each shard. Otherwise it produces a constant value. +**Syntax** + +``` sql +uptime() +``` + +**Returned value** + +- Time value of seconds. + +Type: [UInt32](/docs/en/sql-reference/data-types/int-uint.md). + +**Example** + +Query: + +``` sql +SELECT uptime() as Uptime; +``` + +Result: + +``` response +┌─Uptime─┐ +│ 55867 │ +└────────┘ +``` + ## version() Returns the current version of ClickHouse as a string in the form of: @@ -3132,3 +3212,17 @@ Result: │ (616.2931945826209,108.8825,115.6175) │ └───────────────────────────────────────┘ ``` + +## getClientHTTPHeader + +Get the value of an HTTP header. + +If there is no such header or the current request is not performed via the HTTP interface, the function returns an empty string. +Certain HTTP headers (e.g., `Authentication` and `X-ClickHouse-*`) are restricted. + +The function requires the setting `allow_get_client_http_header` to be enabled. +The setting is not enabled by default for security reasons, because some headers, such as `Cookie`, could contain sensitive info. + +HTTP headers are case sensitive for this function. + +If the function is used in the context of a distributed query, it returns non-empty result only on the initiator node. diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index b4e2adbed3c..d4df3e0479a 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -99,7 +99,7 @@ Alias: `OCTET_LENGTH` Returns the length of a string in Unicode code points (not: in bytes or characters). It assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined. Alias: -- `CHAR_LENGTH`` +- `CHAR_LENGTH` - `CHARACTER_LENGTH` ## leftPad @@ -260,8 +260,36 @@ Alias: `lcase` Converts the ASCII Latin symbols in a string to uppercase. +**Syntax** + +``` sql +upper(input) +``` + Alias: `ucase` +**Parameters** + +- `input`: A string type [String](/docs/en/sql-reference/data-types/string.md). + +**Returned value** + +- A [String](/docs/en/sql-reference/data-types/string.md) data type value. + +**Examples** + +Query: + +``` sql +SELECT upper('value') as Upper; +``` + +``` response +┌─Upper─┐ +│ VALUE │ +└───────┘ +``` + ## lowerUTF8 Converts a string to lowercase, assuming that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined. @@ -278,6 +306,34 @@ Does not detect the language, e.g. for Turkish the result might not be exactly c If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point. +**Syntax** + +``` sql +upperUTF8(input) +``` + +**Parameters** + +- `input`: A string type [String](/docs/en/sql-reference/data-types/string.md). + +**Returned value** + +- A [String](/docs/en/sql-reference/data-types/string.md) data type value. + +**Example** + +Query: + +``` sql +SELECT upperUTF8('München') as Upperutf8; +``` + +``` response +┌─Upperutf8─┐ +│ MÜNCHEN │ +└───────────┘ +``` + ## isValidUTF8 Returns 1, if the set of bytes constitutes valid UTF-8-encoded text, otherwise 0. diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index c7bd16cad4a..60fe286de25 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -193,3 +193,33 @@ Result: ## translateUTF8 Like [translate](#translate) but assumes `s`, `from` and `to` are UTF-8 encoded strings. + +**Syntax** + +``` sql +translateUTF8(s, from, to) +``` + +**Parameters** + +- `s`: A string type [String](/docs/en/sql-reference/data-types/string.md). +- `from`: A string type [String](/docs/en/sql-reference/data-types/string.md). +- `to`: A string type [String](/docs/en/sql-reference/data-types/string.md). + +**Returned value** + +- `s`: A string type [String](/docs/en/sql-reference/data-types/string.md). + +**Examples** + +Query: + +``` sql +SELECT translateUTF8('Münchener Straße', 'üß', 'us') AS res; +``` + +``` response +┌─res──────────────┐ +│ Munchener Strase │ +└──────────────────┘ +``` diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 1b03f220db2..f7e56e73520 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -6,14 +6,17 @@ sidebar_label: Searching in Strings # Functions for Searching in Strings -All functions in this section search by default case-sensitively. Case-insensitive search is usually provided by separate function variants. -Note that case-insensitive search follows the lowercase-uppercase rules of the English language. E.g. Uppercased `i` in English language is -`I` whereas in Turkish language it is `İ` - results for languages other than English may be unexpected. +All functions in this section search case-sensitively by default. Case-insensitive search is usually provided by separate function variants. -Functions in this section also assume that the searched string and the search string are single-byte encoded text. If this assumption is +:::note +Case-insensitive search follows the lowercase-uppercase rules of the English language. E.g. Uppercased `i` in the English language is +`I` whereas in the Turkish language it is `İ` - results for languages other than English may be unexpected. +::: + +Functions in this section also assume that the searched string (referred to in this section as `haystack`) and the search string (referred to in this section as `needle`) are single-byte encoded text. If this assumption is violated, no exception is thrown and results are undefined. Search with UTF-8 encoded strings is usually provided by separate function variants. Likewise, if a UTF-8 function variant is used and the input strings are not UTF-8 encoded text, no exception is thrown and the -results are undefined. Note that no automatic Unicode normalization is performed, you can use the +results are undefined. Note that no automatic Unicode normalization is performed, however you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that. [General strings functions](string-functions.md) and [functions for replacing in strings](string-replace-functions.md) are described separately. @@ -54,6 +57,8 @@ Type: `Integer`. **Examples** +Query: + ``` sql SELECT position('Hello, world!', '!'); ``` @@ -68,12 +73,16 @@ Result: Example with `start_pos` argument: +Query: + ``` sql SELECT position('Hello, world!', 'o', 1), position('Hello, world!', 'o', 7) ``` +Result: + ``` text ┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐ │ 5 │ 9 │ @@ -82,6 +91,8 @@ SELECT Example for `needle IN haystack` syntax: +Query: + ```sql SELECT 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); ``` @@ -96,6 +107,8 @@ Result: Examples with empty `needle` substring: +Query: + ``` sql SELECT position('abc', ''), @@ -107,6 +120,8 @@ SELECT position('abc', '', 5) ``` +Result: + ``` text ┌─position('abc', '')─┬─position('abc', '', 0)─┬─position('abc', '', 1)─┬─position('abc', '', 2)─┬─position('abc', '', 3)─┬─position('abc', '', 4)─┬─position('abc', '', 5)─┐ │ 1 │ 1 │ 1 │ 2 │ 3 │ 4 │ 0 │ @@ -130,7 +145,23 @@ locate(needle, haystack[, start_pos]) ## positionCaseInsensitive -Like [position](#position) but searches case-insensitively. +A case insensitive invariant of [position](#position). + +**Example** + +Query: + +``` sql +SELECT position('Hello, world!', 'hello'); +``` + +Result: + +``` text +┌─position('Hello, world!', 'hello')─┐ +│ 0 │ +└────────────────────────────────────┘ +``` ## positionUTF8 @@ -140,6 +171,8 @@ Like [position](#position) but assumes `haystack` and `needle` are UTF-8 encoded Function `positionUTF8` correctly counts character `ö` (represented by two points) as a single Unicode codepoint: +Query: + ``` sql SELECT positionUTF8('Motörhead', 'r'); ``` @@ -173,14 +206,17 @@ multiSearchAllPositions(haystack, [needle1, needle2, ..., needleN]) **Arguments** - `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substrings to be searched. Array +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). **Returned values** -- Array of the starting position in bytes and counting from 1 (if the substring was found) or 0 (if the substring was not found) +- Array of the starting position in bytes and counting from 1, if the substring was found. +- 0, if the substring was not found. **Example** +Query: + ``` sql SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']); ``` @@ -192,45 +228,535 @@ Result: │ [0,13,0] │ └───────────────────────────────────────────────────────────────────┘ ``` +## multiSearchAllPositionsCaseInsensitive -## multiSearchAllPositionsUTF8 - -Like [multiSearchAllPositions](#multiSearchAllPositions) but assumes `haystack` and the `needle`-s are UTF-8 encoded strings. - -## multiSearchFirstPosition - -Like `position` but returns the leftmost offset in a `haystack` string which matches any of multiple `needle` strings. - -Functions `multiSearchFirstPositionCaseInsensitive`, `multiSearchFirstPositionUTF8` and `multiSearchFirstPositionCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. +Like [multiSearchAllPositions](#multisearchallpositions) but ignores case. **Syntax** ```sql -multiSearchFirstPosition(haystack, \[needle1, needle2, …, needlen\]) +multiSearchAllPositionsCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- Array of the starting position in bytes and counting from 1 (if the substring was found). +- 0 if the substring was not found. + +**Example** + +Query: + +```sql +SELECT multiSearchAllPositionsCaseInsensitive('ClickHouse',['c','h']); +``` + +Result: + +```response +["1","6"] +``` + +## multiSearchAllPositionsUTF8 + +Like [multiSearchAllPositions](#multiSearchAllPositions) but assumes `haystack` and the `needle` substrings are UTF-8 encoded strings. + +**Syntax** + +```sql +multiSearchAllPositionsUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- Array of the starting position in bytes and counting from 1 (if the substring was found). +- 0 if the substring was not found. + +**Example** + +Given `ClickHouse` as a UTF-8 string, find the positions of `C` (`\x43`) and `H` (`\x48`). + +Query: + +```sql +SELECT multiSearchAllPositionsUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x43','\x48']); +``` + +Result: + +```response +["1","6"] +``` + +## multiSearchAllPositionsCaseInsensitiveUTF8 + +Like [multiSearchAllPositionsUTF8](#multisearchallpositionsutf8) but ignores case. + +**Syntax** + +```sql +multiSearchAllPositionsCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- Array of the starting position in bytes and counting from 1 (if the substring was found). +- 0 if the substring was not found. + +**Example** + +Given `ClickHouse` as a UTF-8 string, find the positions of `c` (`\x63`) and `h` (`\x68`). + +Query: + +```sql +SELECT multiSearchAllPositionsCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x63','\x68']); +``` + +Result: + +```response +["1","6"] +``` + +## multiSearchFirstPosition + +Like [`position`](#position) but returns the leftmost offset in a `haystack` string which matches any of multiple `needle` strings. + +Functions [`multiSearchFirstPositionCaseInsensitive`](#multiSearchFirstPositionCaseInsensitive), [`multiSearchFirstPositionUTF8`](#multiSearchFirstPositionUTF8) and [`multiSearchFirstPositionCaseInsensitiveUTF8`](#multiSearchFirstPositionCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function. + +**Syntax** + +```sql +multiSearchFirstPosition(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings. +- 0, if there was no match. + +**Example** + +Query: + +```sql +SELECT multiSearchFirstPosition('Hello World',['llo', 'Wor', 'ld']); +``` + +Result: + +```response +3 +``` + +## multiSearchFirstPositionCaseInsensitive + +Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but ignores case. + +**Syntax** + +```sql +multiSearchFirstPositionCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings. +- 0, if there was no match. + +**Example** + +Query: + +```sql +SELECT multiSearchFirstPositionCaseInsensitive('HELLO WORLD',['wor', 'ld', 'ello']); +``` + +Result: + +```response +2 +``` + +## multiSearchFirstPositionUTF8 + +Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings. + +**Syntax** + +```sql +multiSearchFirstPositionUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings. +- 0, if there was no match. + +**Example** + +Find the leftmost offset in UTF-8 string `hello world` which matches any of the given needles. + +Query: + +```sql +SELECT multiSearchFirstPositionUTF8('\x68\x65\x6c\x6c\x6f\x20\x77\x6f\x72\x6c\x64',['wor', 'ld', 'ello']); +``` + +Result: + +```response +2 +``` + +## multiSearchFirstPositionCaseInsensitiveUTF8 + +Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings and ignores case. + +**Syntax** + +```sql +multiSearchFirstPositionCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings, ignoring case. +- 0, if there was no match. + +**Example** + +Find the leftmost offset in UTF-8 string `HELLO WORLD` which matches any of the given needles. + +Query: + +```sql +SELECT multiSearchFirstPositionCaseInsensitiveUTF8('\x48\x45\x4c\x4c\x4f\x20\x57\x4f\x52\x4c\x44',['wor', 'ld', 'ello']); +``` + +Result: + +```response +2 ``` ## multiSearchFirstIndex Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. -Functions `multiSearchFirstIndexCaseInsensitive`, `multiSearchFirstIndexUTF8` and `multiSearchFirstIndexCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. +Functions [`multiSearchFirstIndexCaseInsensitive`](#multiSearchFirstIndexCaseInsensitive), [`multiSearchFirstIndexUTF8`](#multiSearchFirstIndexUTF8) and [`multiSearchFirstIndexCaseInsensitiveUTF8`](#multiSearchFirstIndexCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** ```sql -multiSearchFirstIndex(haystack, \[needle1, needle2, …, needlen\]) +multiSearchFirstIndex(haystack, [needle1, needle2, ..., needleN]) +``` +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- index (starting from 1) of the leftmost found needle. +- 0, if there was no match. + +**Example** + +Query: + +```sql +SELECT multiSearchFirstIndex('Hello World',['World','Hello']); ``` -## multiSearchAny {#multisearchany} +Result: + +```response +1 +``` + +## multiSearchFirstIndexCaseInsensitive + +Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. Ignores case. + +**Syntax** + +```sql +multiSearchFirstIndexCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- index (starting from 1) of the leftmost found needle. +- 0, if there was no match. + +**Example** + +Query: + +```sql +SELECT multiSearchFirstIndexCaseInsensitive('hElLo WoRlD',['World','Hello']); +``` + +Result: + +```response +1 +``` + +## multiSearchFirstIndexUTF8 + +Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. Assumes `haystack` and `needle` are UTF-8 encoded strings. + +**Syntax** + +```sql +multiSearchFirstIndexUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- index (starting from 1) of the leftmost found needle. +- 0, if there was no match. + +**Example** + +Given `Hello World` as a UTF-8 string, find the first index of UTF-8 strings `Hello` and `World`. + +Query: + +```sql +SELECT multiSearchFirstIndexUTF8('\x48\x65\x6c\x6c\x6f\x20\x57\x6f\x72\x6c\x64',['\x57\x6f\x72\x6c\x64','\x48\x65\x6c\x6c\x6f']); +``` + +Result: + +```response +1 +``` + +## multiSearchFirstIndexCaseInsensitiveUTF8 + +Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise. Assumes `haystack` and `needle` are UTF-8 encoded strings. Ignores case. + +**Syntax** + +```sql +multiSearchFirstIndexCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- index (starting from 1) of the leftmost found needle. +- 0, if there was no match. + +**Example** + +Given `HELLO WORLD` as a UTF-8 string, find the first index of UTF-8 strings `hello` and `world`. + +Query: + +```sql +SELECT multiSearchFirstIndexCaseInsensitiveUTF8('\x48\x45\x4c\x4c\x4f\x20\x57\x4f\x52\x4c\x44',['\x68\x65\x6c\x6c\x6f','\x77\x6f\x72\x6c\x64']); +``` + +Result: + +```response +1 +``` + +## multiSearchAny Returns 1, if at least one string needlei matches the string `haystack` and 0 otherwise. -Functions `multiSearchAnyCaseInsensitive`, `multiSearchAnyUTF8` and `multiSearchAnyCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. +Functions [`multiSearchAnyCaseInsensitive`](#multiSearchAnyCaseInsensitive), [`multiSearchAnyUTF8`](#multiSearchAnyUTF8) and []`multiSearchAnyCaseInsensitiveUTF8`](#multiSearchAnyCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** ```sql -multiSearchAny(haystack, \[needle1, needle2, …, needlen\]) +multiSearchAny(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- 1, if there was at least one match. +- 0, if there was not at least one match. + +**Example** + +Query: + +```sql +SELECT multiSearchAny('ClickHouse',['C','H']); +``` + +Result: + +```response +1 +``` + +## multiSearchAnyCaseInsensitive + +Like [multiSearchAny](#multisearchany) but ignores case. + +**Syntax** + +```sql +multiSearchAnyCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- 1, if there was at least one case-insensitive match. +- 0, if there was not at least one case-insensitive match. + +**Example** + +Query: + +```sql +SELECT multiSearchAnyCaseInsensitive('ClickHouse',['c','h']); +``` + +Result: + +```response +1 +``` + +## multiSearchAnyUTF8 + +Like [multiSearchAny](#multisearchany) but assumes `haystack` and the `needle` substrings are UTF-8 encoded strings. + +*Syntax** + +```sql +multiSearchAnyUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- 1, if there was at least one match. +- 0, if there was not at least one match. + +**Example** + +Given `ClickHouse` as a UTF-8 string, check if there are any `C` ('\x43') or `H` ('\x48') letters in the word. + +Query: + +```sql +SELECT multiSearchAnyUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x43','\x48']); +``` + +Result: + +```response +1 +``` + +## multiSearchAnyCaseInsensitiveUTF8 + +Like [multiSearchAnyUTF8](#multiSearchAnyUTF8) but ignores case. + +*Syntax** + +```sql +multiSearchAnyCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN]) +``` + +**Parameters** + +- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md) + +**Returned value** + +- 1, if there was at least one case-insensitive match. +- 0, if there was not at least one case-insensitive match. + +**Example** + +Given `ClickHouse` as a UTF-8 string, check if there is any letter `h`(`\x68`) in the word, ignoring case. + +Query: + +```sql +SELECT multiSearchAnyCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x68']); +``` + +Result: + +```response +1 ``` ## match {#match} @@ -479,9 +1005,9 @@ Alias: `haystack NOT ILIKE pattern` (operator) ## ngramDistance -Calculates the 4-gram distance between a `haystack` string and a `needle` string. For that, it counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns a Float32 between 0 and 1. The smaller the result is, the more strings are similar to each other. Throws an exception if constant `needle` or `haystack` arguments are more than 32Kb in size. If any of non-constant `haystack` or `needle` arguments is more than 32Kb in size, the distance is always 1. +Calculates the 4-gram distance between a `haystack` string and a `needle` string. For this, it counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns a [Float32](../../sql-reference/data-types/float.md/#float32-float64) between 0 and 1. The smaller the result is, the more similar the strings are to each other. -Functions `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. +Functions [`ngramDistanceCaseInsensitive`](#ngramdistancecaseinsensitive), [`ngramDistanceUTF8`](#ngramdistanceutf8), [`ngramDistanceCaseInsensitiveUTF8`](#ngramdistancecaseinsensitiveutf8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** @@ -489,15 +1015,170 @@ Functions `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseIns ngramDistance(haystack, needle) ``` +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Implementation details** + +This function will throw an exception if constant `needle` or `haystack` arguments are more than 32Kb in size. If any non-constant `haystack` or `needle` arguments are more than 32Kb in size, then the distance is always 1. + +**Examples** + +The more similar two strings are to each other, the closer the result will be to 0 (identical). + +Query: + +```sql +SELECT ngramDistance('ClickHouse','ClickHouse!'); +``` + +Result: + +```response +0.06666667 +``` + +The less similar two strings are to each, the larger the result will be. + + +Query: + +```sql +SELECT ngramDistance('ClickHouse','House'); +``` + +Result: + +```response +0.5555556 +``` + +## ngramDistanceCaseInsensitive + +Provides a case-insensitive variant of [ngramDistance](#ngramdistance). + +**Syntax** + +```sql +ngramDistanceCaseInsensitive(haystack, needle) +``` + +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Examples** + +With [ngramDistance](#ngramdistance) differences in case will affect the similarity value: + +Query: + +```sql +SELECT ngramDistance('ClickHouse','clickhouse'); +``` + +Result: + +```response +0.71428573 +``` + +With [ngramDistanceCaseInsensitive](#ngramdistancecaseinsensitive) case is ignored so two identical strings differing only in case will now return a low similarity value: + +Query: + +```sql +SELECT ngramDistanceCaseInsensitive('ClickHouse','clickhouse'); +``` + +Result: + +```response +0 +``` + +## ngramDistanceUTF8 + +Provides a UTF-8 variant of [ngramDistance](#ngramdistance). Assumes that `needle` and `haystack` strings are UTF-8 encoded strings. + +**Syntax** + +```sql +ngramDistanceUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Example** + +Query: + +```sql +SELECT ngramDistanceUTF8('abcde','cde'); +``` + +Result: + +```response +0.5 +``` + +## ngramDistanceCaseInsensitiveUTF8 + +Provides a case-insensitive variant of [ngramDistanceUTF8](#ngramdistanceutf8). + +**Syntax** + +```sql +ngramDistanceCaseInsensitiveUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Example** + +Query: + +```sql +SELECT ngramDistanceCaseInsensitiveUTF8('abcde','CDE'); +``` + +Result: + +```response +0.5 +``` + ## ngramSearch -Like `ngramDistance` but calculates the non-symmetric difference between a `needle` string and a `haystack` string, i.e. the number of n-grams from needle minus the common number of n-grams normalized by the number of `needle` n-grams. Returns a Float32 between 0 and 1. The bigger the result is, the more likely `needle` is in the `haystack`. This function is useful for fuzzy string search. Also see function `soundex`. +Like `ngramDistance` but calculates the non-symmetric difference between a `needle` string and a `haystack` string, i.e. the number of n-grams from the needle minus the common number of n-grams normalized by the number of `needle` n-grams. Returns a [Float32](../../sql-reference/data-types/float.md/#float32-float64) between 0 and 1. The bigger the result is, the more likely `needle` is in the `haystack`. This function is useful for fuzzy string search. Also see function [`soundex`](../../sql-reference/functions/string-functions#soundex). -Functions `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. - -:::note -The UTF-8 variants use the 3-gram distance. These are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables – collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function – we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one – this works for Latin and mostly for all Cyrillic letters. -::: +Functions [`ngramSearchCaseInsensitive`](#ngramsearchcaseinsensitive), [`ngramSearchUTF8`](#ngramsearchutf8), [`ngramSearchCaseInsensitiveUTF8`](#ngramsearchcaseinsensitiveutf8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** @@ -505,6 +1186,140 @@ The UTF-8 variants use the 3-gram distance. These are not perfectly fair n-gram ngramSearch(haystack, needle) ``` +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Implementation details** + +:::note +The UTF-8 variants use the 3-gram distance. These are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables – collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function – we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one – this works for Latin and mostly for all Cyrillic letters. +::: + +**Example** + +Query: + +```sql +SELECT ngramSearch('Hello World','World Hello'); +``` + +Result: + +```response +0.5 +``` + +## ngramSearchCaseInsensitive + +Provides a case-insensitive variant of [ngramSearch](#ngramSearch). + +**Syntax** + +```sql +ngramSearchCaseInsensitive(haystack, needle) +``` + +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +The bigger the result is, the more likely `needle` is in the `haystack`. + +**Example** + +Query: + +```sql +SELECT ngramSearchCaseInsensitive('Hello World','hello'); +``` + +Result: + +```response +1 +``` + +## ngramSearchUTF8 + +Provides a UTF-8 variant of [ngramSearch](#ngramsearch) in which `needle` and `haystack` are assumed to be UTF-8 encoded strings. + +**Syntax** + +```sql +ngramSearchUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +The bigger the result is, the more likely `needle` is in the `haystack`. + +**Example** + +Query: + +```sql +SELECT ngramSearchUTF8('абвгдеёжз', 'гдеёзд'); +``` + +Result: + +```response +0.5 +``` + +## ngramSearchCaseInsensitiveUTF8 + +Provides a case-insensitive variant of [ngramSearchUTF8](#ngramsearchutf8) in which `needle` and `haystack`. + +**Syntax** + +```sql +ngramSearchCaseInsensitiveUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +The bigger the result is, the more likely `needle` is in the `haystack`. + +**Example** + +Query: + +```sql +SELECT ngramSearchCaseInsensitiveUTF8('абвГДЕёжз', 'АбвгдЕЁжз'); +``` + +Result: + +```response +0.57142854 +``` + ## countSubstrings Returns how often substring `needle` occurs in string `haystack`. @@ -610,7 +1425,7 @@ Like `countMatches(haystack, pattern)` but matching ignores the case. ## regexpExtract -Extracts the first string in haystack that matches the regexp pattern and corresponds to the regex group index. +Extracts the first string in `haystack` that matches the regexp pattern and corresponds to the regex group index. **Syntax** @@ -652,7 +1467,7 @@ Result: ## hasSubsequence -Returns 1 if needle is a subsequence of haystack, or 0 otherwise. +Returns 1 if `needle` is a subsequence of `haystack`, or 0 otherwise. A subsequence of a string is a sequence that can be derived from the given string by deleting zero or more elements without changing the order of the remaining elements. @@ -676,8 +1491,10 @@ Type: `UInt8`. **Examples** +Query: + ``` sql -SELECT hasSubsequence('garbage', 'arg') ; +SELECT hasSubsequence('garbage', 'arg'); ``` Result: @@ -692,10 +1509,263 @@ Result: Like [hasSubsequence](#hasSubsequence) but searches case-insensitively. +**Syntax** + +``` sql +hasSubsequenceCaseInsensitive(haystack, needle) +``` + +**Arguments** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Subsequence to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). + +**Returned values** + +- 1, if needle is a subsequence of haystack. +- 0, otherwise. + +Type: `UInt8`. + +**Examples** + +Query: + +``` sql +SELECT hasSubsequenceCaseInsensitive('garbage', 'ARG'); +``` + +Result: + +``` text +┌─hasSubsequenceCaseInsensitive('garbage', 'ARG')─┐ +│ 1 │ +└─────────────────────────────────────────────────┘ +``` + ## hasSubsequenceUTF8 Like [hasSubsequence](#hasSubsequence) but assumes `haystack` and `needle` are UTF-8 encoded strings. +**Syntax** + +``` sql +hasSubsequenceUTF8(haystack, needle) +``` + +**Arguments** + +- `haystack` — String in which the search is performed. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Subsequence to be searched. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). + +**Returned values** + +- 1, if needle is a subsequence of haystack. +- 0, otherwise. + +Type: `UInt8`. + +Query: + +**Examples** + +``` sql +select hasSubsequenceUTF8('ClickHouse - столбцовая система управления базами данных', 'система'); +``` + +Result: + +``` text +┌─hasSubsequenceUTF8('ClickHouse - столбцовая система управления базами данных', 'система')─┐ +│ 1 │ +└───────────────────────────────────────────────────────────────────────────────────────────┘ +``` + ## hasSubsequenceCaseInsensitiveUTF8 Like [hasSubsequenceUTF8](#hasSubsequenceUTF8) but searches case-insensitively. + +**Syntax** + +``` sql +hasSubsequenceCaseInsensitiveUTF8(haystack, needle) +``` + +**Arguments** + +- `haystack` — String in which the search is performed. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Subsequence to be searched. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). + +**Returned values** + +- 1, if needle is a subsequence of haystack. +- 0, otherwise. + +Type: `UInt8`. + +**Examples** + +Query: + +``` sql +select hasSubsequenceCaseInsensitiveUTF8('ClickHouse - столбцовая система управления базами данных', 'СИСТЕМА'); +``` + +Result: + +``` text +┌─hasSubsequenceCaseInsensitiveUTF8('ClickHouse - столбцовая система управления базами данных', 'СИСТЕМА')─┐ +│ 1 │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## hasToken + +Returns 1 if a given token is present in a haystack, or 0 otherwise. + +**Syntax** + +```sql +hasToken(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, if the token is not present. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + +Query: + +```sql +SELECT hasToken('Hello World','Hello'); +``` + +```response +1 +``` + +## hasTokenOrNull + +Returns 1 if a given token is present, 0 if not present, and null if the token is ill-formed. + +**Syntax** + +```sql +hasTokenOrNull(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, if the token is not present in the haystack. +- null, if the token is ill-formed. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + +Where `hasToken` would throw an error for an ill-formed token, `hasTokenOrNull` returns `null` for an ill-formed token. + +Query: + +```sql +SELECT hasTokenOrNull('Hello World','Hello,World'); +``` + +```response +null +``` + +## hasTokenCaseInsensitive + +Returns 1 if a given token is present in a haystack, 0 otherwise. Ignores case. + +**Syntax** + +```sql +hasTokenCaseInsensitive(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, otherwise. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + +Query: + +```sql +SELECT hasTokenCaseInsensitive('Hello World','hello'); +``` + +```response +1 +``` + +## hasTokenCaseInsensitiveOrNull + +Returns 1 if a given token is present in a haystack, 0 otherwise. Ignores case and returns null if the token is ill-formed. + +**Syntax** + +```sql +hasTokenCaseInsensitiveOrNull(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, if token is not present. +- null, if the token is ill-formed. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + + +Where `hasTokenCaseInsensitive` would throw an error for an ill-formed token, `hasTokenCaseInsensitiveOrNull` returns `null` for an ill-formed token. + +Query: + +```sql +SELECT hasTokenCaseInsensitiveOrNull('Hello World','hello,world'); +``` + +```response +null +``` \ No newline at end of file diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index b089de67e98..b3cec1206b8 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -584,6 +584,278 @@ SELECT tupleConcat((1, 2), (3, 4), (true, false)) AS res └──────────────────────┘ ``` +## tupleIntDiv + +Does integer division of a tuple of numerators and a tuple of denominators, and returns a tuple of the quotients. + +**Syntax** + +```sql +tupleIntDiv(tuple_num, tuple_div) +``` + +**Parameters** + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `tuple_div`: Tuple of divisor values. [Tuple](../data-types/tuple) of numeric type. + +**Returned value** + +- Tuple of the quotients of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of integer values. + +**Implementation details** + +- If either `tuple_num` or `tuple_div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor. +- An error will be thrown for division by 0. + +**Examples** + +Query: + +``` sql +SELECT tupleIntDiv((15, 10, 5), (5, 5, 5)); +``` + +Result: + +``` text +┌─tupleIntDiv((15, 10, 5), (5, 5, 5))─┐ +│ (3,2,1) │ +└─────────────────────────────────────┘ +``` + +Query: + +``` sql +SELECT tupleIntDiv((15, 10, 5), (5.5, 5.5, 5.5)); +``` + +Result: + +``` text +┌─tupleIntDiv((15, 10, 5), (5.5, 5.5, 5.5))─┐ +│ (2,1,0) │ +└───────────────────────────────────────────┘ +``` + +## tupleIntDivOrZero + +Like [tupleIntDiv](#tupleintdiv) it does integer division of a tuple of numerators and a tuple of denominators, and returns a tuple of the quotients. It does not throw an error for 0 divisors, but rather returns the quotient as 0. + +**Syntax** + +```sql +tupleIntDivOrZero(tuple_num, tuple_div) +``` + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `tuple_div`: Tuple of divisor values. [Tuple](../data-types/tuple) of numeric type. + +**Returned value** + +- Tuple of the quotients of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of integer values. +- Returns 0 for quotients where the divisor is 0. + +**Implementation details** + +- If either `tuple_num` or `tuple_div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor as in [tupleIntDiv](#tupleintdiv). + +**Examples** + +Query: + +``` sql +SELECT tupleIntDivOrZero((5, 10, 15), (0, 0, 0)); +``` + +Result: + +``` text +┌─tupleIntDivOrZero((5, 10, 15), (0, 0, 0))─┐ +│ (0,0,0) │ +└───────────────────────────────────────────┘ +``` + +## tupleIntDivByNumber + +Does integer division of a tuple of numerators by a given denominator, and returns a tuple of the quotients. + +**Syntax** + +```sql +tupleIntDivByNumber(tuple_num, div) +``` + +**Parameters** + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `div`: The divisor value. [Numeric](../data-types/int-uint.md) type. + +**Returned value** + +- Tuple of the quotients of `tuple_num` and `div`. [Tuple](../data-types/tuple) of integer values. + +**Implementation details** + +- If either `tuple_num` or `div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor. +- An error will be thrown for division by 0. + +**Examples** + +Query: + +``` sql +SELECT tupleIntDivByNumber((15, 10, 5), 5); +``` + +Result: + +``` text +┌─tupleIntDivByNumber((15, 10, 5), 5)─┐ +│ (3,2,1) │ +└─────────────────────────────────────┘ +``` + +Query: + +``` sql +SELECT tupleIntDivByNumber((15.2, 10.7, 5.5), 5.8); +``` + +Result: + +``` text +┌─tupleIntDivByNumber((15.2, 10.7, 5.5), 5.8)─┐ +│ (2,1,0) │ +└─────────────────────────────────────────────┘ +``` + +## tupleIntDivOrZeroByNumber + +Like [tupleIntDivByNumber](#tupleintdivbynumber) it does integer division of a tuple of numerators by a given denominator, and returns a tuple of the quotients. It does not throw an error for 0 divisors, but rather returns the quotient as 0. + +**Syntax** + +```sql +tupleIntDivOrZeroByNumber(tuple_num, div) +``` + +**Parameters** + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `div`: The divisor value. [Numeric](../data-types/int-uint.md) type. + +**Returned value** + +- Tuple of the quotients of `tuple_num` and `div`. [Tuple](../data-types/tuple) of integer values. +- Returns 0 for quotients where the divisor is 0. + +**Implementation details** + +- If either `tuple_num` or `div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor as in [tupleIntDivByNumber](#tupleintdivbynumber). + +**Examples** + +Query: + +``` sql +SELECT tupleIntDivOrZeroByNumber((15, 10, 5), 5); +``` + +Result: + +``` text +┌─tupleIntDivOrZeroByNumber((15, 10, 5), 5)─┐ +│ (3,2,1) │ +└───────────────────────────────────────────┘ +``` + +Query: + +``` sql +SELECT tupleIntDivOrZeroByNumber((15, 10, 5), 0) +``` + +Result: + +``` text +┌─tupleIntDivOrZeroByNumber((15, 10, 5), 0)─┐ +│ (0,0,0) │ +└───────────────────────────────────────────┘ +``` + +## tupleModulo + +Returns a tuple of the moduli (remainders) of division operations of two tuples. + +**Syntax** + +```sql +tupleModulo(tuple_num, tuple_mod) +``` + +**Parameters** + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `tuple_div`: Tuple of modulus values. [Tuple](../data-types/tuple) of numeric type. + +**Returned value** + +- Tuple of the remainders of division of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of non-zero integer values. +- An error is thrown for division by zero. + +**Examples** + +Query: + +``` sql +SELECT tupleModulo((15, 10, 5), (5, 3, 2)); +``` + +Result: + +``` text +┌─tupleModulo((15, 10, 5), (5, 3, 2))─┐ +│ (0,1,1) │ +└─────────────────────────────────────┘ +``` + +## tupleModuloByNumber + +Returns a tuple of the moduli (remainders) of division operations of a tuple and a given divisor. + +**Syntax** + +```sql +tupleModuloByNumber(tuple_num, div) +``` + +**Parameters** + +- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type. +- `div`: The divisor value. [Numeric](../data-types/int-uint.md) type. + +**Returned value** + +- Tuple of the remainders of division of `tuple_num` and `div`. [Tuple](../data-types/tuple) of non-zero integer values. +- An error is thrown for division by zero. + +**Examples** + +Query: + +``` sql +SELECT tupleModuloByNumber((15, 10, 5), 2); +``` + +Result: + +``` text +┌─tupleModuloByNumber((15, 10, 5), 2)─┐ +│ (1,0,1) │ +└─────────────────────────────────────┘ +``` + ## Distance functions All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md). diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index ac81815b47f..a0b0170721c 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -128,9 +128,9 @@ Returns the part of the domain that includes top-level subdomains up to the “f For example: -- `cutToFirstSignificantSubdomain('https://news.clickhouse.com.tr/') = 'clickhouse.com.tr'`. -- `cutToFirstSignificantSubdomain('www.tr') = 'www.tr'`. -- `cutToFirstSignificantSubdomain('tr') = ''`. +- `cutToFirstSignificantSubdomainWithWWW('https://news.clickhouse.com.tr/') = 'clickhouse.com.tr'`. +- `cutToFirstSignificantSubdomainWithWWW('www.tr') = 'www.tr'`. +- `cutToFirstSignificantSubdomainWithWWW('tr') = ''`. ### cutToFirstSignificantSubdomainCustom diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md index dc6668c7983..7961315c193 100644 --- a/docs/en/sql-reference/statements/alter/index.md +++ b/docs/en/sql-reference/statements/alter/index.md @@ -56,7 +56,9 @@ Entries for finished mutations are not deleted right away (the number of preserv For non-replicated tables, all `ALTER` queries are performed synchronously. For replicated tables, the query just adds instructions for the appropriate actions to `ZooKeeper`, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas. -For all `ALTER` queries, you can use the [alter_sync](/docs/en/operations/settings/settings.md/#alter-sync) setting to set up waiting. +For `ALTER` queries that creates mutations (e.g.: including, but not limited to `UPDATE`, `DELETE`, `MATERIALIZE INDEX`, `MATERIALIZE PROJECTION`, `MATERIALIZE COLUMN`, `APPLY DELETED MASK`, `CLEAR STATISTIC`, `MATERIALIZE STATISTIC`) the synchronicity is defined by the [mutations_sync](/docs/en/operations/settings/settings.md/#mutations_sync) setting. + +For other `ALTER` queries which only modify the metadata, you can use the [alter_sync](/docs/en/operations/settings/settings.md/#alter-sync) setting to set up waiting. You can specify how long (in seconds) to wait for inactive replicas to execute all `ALTER` queries with the [replication_wait_for_inactive_replica_timeout](/docs/en/operations/settings/settings.md/#replication-wait-for-inactive-replica-timeout) setting. @@ -64,8 +66,6 @@ You can specify how long (in seconds) to wait for inactive replicas to execute a For all `ALTER` queries, if `alter_sync = 2` and some replicas are not active for more than the time, specified in the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. ::: -For `ALTER TABLE ... UPDATE|DELETE|MATERIALIZE INDEX|MATERIALIZE PROJECTION|MATERIALIZE COLUMN` queries the synchronicity is defined by the [mutations_sync](/docs/en/operations/settings/settings.md/#mutations_sync) setting. - ## Related content - Blog: [Handling Updates and Deletes in ClickHouse](https://clickhouse.com/blog/handling-updates-and-deletes-in-clickhouse) diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index ce5cecf6fd6..941dc000a02 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -133,6 +133,8 @@ For the query to run successfully, the following conditions must be met: - Both tables must have the same indices and projections. - Both tables must have the same storage policy. +If both tables have the same storage policy, use hardlink to attach partition. Otherwise, use copying the data to attach partition. + ## REPLACE PARTITION ``` sql diff --git a/docs/en/sql-reference/statements/alter/view.md b/docs/en/sql-reference/statements/alter/view.md index 59045afdeb6..e063b27424e 100644 --- a/docs/en/sql-reference/statements/alter/view.md +++ b/docs/en/sql-reference/statements/alter/view.md @@ -8,7 +8,7 @@ sidebar_label: VIEW You can modify `SELECT` query that was specified when a [materialized view](../create/view.md#materialized) was created with the `ALTER TABLE … MODIFY QUERY` statement without interrupting ingestion process. -This command is created to change materialized view created with `TO [db.]name` clause. It does not change the structure of the underling storage table and it does not change the columns' definition of the materialized view, because of this the application of this command is very limited for materialized views are created without `TO [db.]name` clause. +This command is created to change materialized view created with `TO [db.]name` clause. It does not change the structure of the underlying storage table and it does not change the columns' definition of the materialized view, because of this the application of this command is very limited for materialized views are created without `TO [db.]name` clause. **Example with TO table** diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 8ed00f625d6..98b849ecf3b 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -20,19 +20,22 @@ DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] [SYNC] ## DROP TABLE -Deletes the table. -In case when `IF EMPTY` clause is specified server will check if table is empty only on replica that received initial query. +Deletes one or more tables. :::tip -Also see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md) +To undo the deletion of a table, please see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md) ::: Syntax: ``` sql -DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY] [db.]name [ON CLUSTER cluster] [SYNC] +DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY] [db1.]name_1[, [db2.]name_2, ...] [ON CLUSTER cluster] [SYNC] ``` +Limitations: +- If the clause `IF EMPTY` is specified, the server checks the emptiness of the table only on the replica which received the query. +- Deleting multiple tables at once is not an atomic operation, i.e. if the deletion of a table fails, subsequent tables will not be deleted. + ## DROP DICTIONARY Deletes the dictionary. diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 294724dfa50..b665ad85a09 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -21,6 +21,35 @@ The queries to terminate are selected from the system.processes table using the Examples: +First, you'll need to get the list of incomplete queries. This SQL query provides them according to those running the longest: + +List from a single ClickHouse node: +``` sql +SELECT + initial_query_id, + query_id, + formatReadableTimeDelta(elapsed) AS time_delta, + query, + * + FROM system.processes + WHERE query ILIKE 'SELECT%' + ORDER BY time_delta DESC; +``` + +List from a ClickHouse cluster: +``` sql +SELECT + initial_query_id, + query_id, + formatReadableTimeDelta(elapsed) AS time_delta, + query, + * + FROM clusterAllReplicas(default, system.processes) + WHERE query ILIKE 'SELECT%' + ORDER BY time_delta DESC; +``` + +Kill the query: ``` sql -- Forcibly terminates all queries with the specified query_id: KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90' @@ -44,6 +73,11 @@ A test query (`TEST`) only checks the user’s rights and displays a list of que ## KILL MUTATION +The presence of long-running or incomplete mutations often indicates that a ClickHouse service is running poorly. The asynchronous nature of mutations can cause them to consume all available resources on a system. You may need to either: + +- Pause all new mutations, `INSERT`s , and `SELECT`s and allow the queue of mutations to complete. +- Or manually kill some of these mutations by sending a `KILL` command. + ``` sql KILL MUTATION [ON CLUSTER cluster] WHERE @@ -57,6 +91,39 @@ A test query (`TEST`) only checks the user’s rights and displays a list of mut Examples: +Get a `count()` of the number of incomplete mutations: + +Count of mutations from a single ClickHouse node: +``` sql +SELECT count(*) +FROM system.mutations +WHERE is_done = 0; +``` + +Count of mutations from a ClickHouse cluster of replicas: +``` sql +SELECT count(*) +FROM clusterAllReplicas('default', system.mutations) +WHERE is_done = 0; +``` + +Query the list of incomplete mutations: + +List of mutations from a single ClickHouse node: +``` sql +SELECT mutation_id, * +FROM system.mutations +WHERE is_done = 0; +``` + +List of mutations from a ClickHouse cluster: +``` sql +SELECT mutation_id, * +FROM clusterAllReplicas('default', system.mutations) +WHERE is_done = 0; +``` + +Kill the mutations as needed: ``` sql -- Cancel and remove all mutations of the single table: KILL MUTATION WHERE database = 'default' AND table = 'table' diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index a128814f072..b35e9426297 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -64,6 +64,14 @@ RELOAD FUNCTIONS [ON CLUSTER cluster_name] RELOAD FUNCTION [ON CLUSTER cluster_name] function_name ``` +## RELOAD ASYNCHRONOUS METRICS + +Re-calculates all [asynchronous metrics](../../operations/system-tables/asynchronous_metrics.md). Since asynchronous metrics are periodically updated based on setting [asynchronous_metrics_update_period_s](../../operations/server-configuration-parameters/settings.md), updating them manually using this statement is typically not necessary. + +```sql +RELOAD ASYNCHRONOUS METRICS [ON CLUSTER cluster_name] +``` + ## DROP DNS CACHE Clears ClickHouse’s internal DNS cache. Sometimes (for old ClickHouse versions) it is necessary to use this command when changing the infrastructure (changing the IP address of another ClickHouse server or the server used by dictionaries). @@ -180,10 +188,16 @@ SYSTEM STOP DISTRIBUTED SENDS [db.] [ON CLUSTER cluster_ Forces ClickHouse to send data to cluster nodes synchronously. If any nodes are unavailable, ClickHouse throws an exception and stops query execution. You can retry the query until it succeeds, which will happen when all nodes are back online. +You can also override some settings via `SETTINGS` clause, this can be useful to avoid some temporary limitations, like `max_concurrent_queries_for_all_users` or `max_memory_usage`. + ``` sql -SYSTEM FLUSH DISTRIBUTED [db.] [ON CLUSTER cluster_name] +SYSTEM FLUSH DISTRIBUTED [db.] [ON CLUSTER cluster_name] [SETTINGS ...] ``` +:::note +Each pending block is stored in disk with settings from the initial INSERT query, so that is why sometimes you may want to override settings. +::: + ### START DISTRIBUTED SENDS Enables background data distribution when inserting data into distributed tables. diff --git a/docs/en/sql-reference/statements/truncate.md b/docs/en/sql-reference/statements/truncate.md index 029815a4392..8cd5a6a1424 100644 --- a/docs/en/sql-reference/statements/truncate.md +++ b/docs/en/sql-reference/statements/truncate.md @@ -23,9 +23,16 @@ You can specify how long (in seconds) to wait for inactive replicas to execute ` If the `alter_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. ::: +## TRUNCATE ALL TABLES +``` sql +TRUNCATE ALL TABLES [IF EXISTS] db [ON CLUSTER cluster] +``` + +Removes all data from all tables in a database. + ## TRUNCATE DATABASE ``` sql -TRUNCATE DATABASE [IF EXISTS] [db.]name [ON CLUSTER cluster] +TRUNCATE DATABASE [IF EXISTS] db [ON CLUSTER cluster] ``` Removes all tables from a database but keeps the database itself. When the clause `IF EXISTS` is omitted, the query returns an error if the database does not exist. diff --git a/docs/en/sql-reference/statements/undrop.md b/docs/en/sql-reference/statements/undrop.md index 40ac1ab4f99..4b138bfe679 100644 --- a/docs/en/sql-reference/statements/undrop.md +++ b/docs/en/sql-reference/statements/undrop.md @@ -13,13 +13,6 @@ a system table called `system.dropped_tables`. If you have a materialized view without a `TO` clause associated with the dropped table, then you will also have to UNDROP the inner table of that view. -:::note -UNDROP TABLE is experimental. To use it add this setting: -```sql -set allow_experimental_undrop_table_query = 1; -``` -::: - :::tip Also see [DROP TABLE](/docs/en/sql-reference/statements/drop.md) ::: @@ -32,60 +25,53 @@ UNDROP TABLE [db.]name [UUID ''] [ON CLUSTER cluster] **Example** -``` sql -set allow_experimental_undrop_table_query = 1; -``` - ```sql -CREATE TABLE undropMe +CREATE TABLE tab ( `id` UInt8 ) ENGINE = MergeTree -ORDER BY id -``` +ORDER BY id; + +DROP TABLE tab; -```sql -DROP TABLE undropMe -``` -```sql SELECT * FROM system.dropped_tables -FORMAT Vertical +FORMAT Vertical; ``` + ```response Row 1: ────── index: 0 database: default -table: undropMe +table: tab uuid: aa696a1a-1d70-4e60-a841-4c80827706cc engine: MergeTree -metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.undropMe.aa696a1a-1d70-4e60-a841-4c80827706cc.sql +metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.tab.aa696a1a-1d70-4e60-a841-4c80827706cc.sql table_dropped_time: 2023-04-05 14:12:12 1 row in set. Elapsed: 0.001 sec. ``` + ```sql -UNDROP TABLE undropMe -``` -```response -Ok. -``` -```sql +UNDROP TABLE tab; + SELECT * FROM system.dropped_tables -FORMAT Vertical -``` +FORMAT Vertical; + ```response Ok. 0 rows in set. Elapsed: 0.001 sec. ``` + ```sql -DESCRIBE TABLE undropMe -FORMAT Vertical +DESCRIBE TABLE tab +FORMAT Vertical; ``` + ```response Row 1: ────── diff --git a/docs/en/sql-reference/table-functions/generate.md b/docs/en/sql-reference/table-functions/generate.md index 3b9b077af49..a78015e9830 100644 --- a/docs/en/sql-reference/table-functions/generate.md +++ b/docs/en/sql-reference/table-functions/generate.md @@ -53,7 +53,7 @@ SELECT * FROM random; └──────────────────────────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ ``` -In combination with [generateRandomStructure](../../sql-reference/functions/other-functions.md#generateRandomStructure): +In combination with [generateRandomStructure](../../sql-reference/functions/other-functions.md#generaterandomstructure): ```sql SELECT * FROM generateRandom(generateRandomStructure(4, 101), 101) LIMIT 3; diff --git a/docs/en/sql-reference/table-functions/generateSeries.md b/docs/en/sql-reference/table-functions/generateSeries.md new file mode 100644 index 00000000000..addf539a959 --- /dev/null +++ b/docs/en/sql-reference/table-functions/generateSeries.md @@ -0,0 +1,8 @@ +--- +slug: /en/sql-reference/table-functions/generateSeries +sidebar_position: 147 +sidebar_label: generateSeries +--- + +### Alias To +[generate_series](generate_series.md) \ No newline at end of file diff --git a/docs/en/sql-reference/table-functions/generate_series.md b/docs/en/sql-reference/table-functions/generate_series.md new file mode 100644 index 00000000000..c5d29369627 --- /dev/null +++ b/docs/en/sql-reference/table-functions/generate_series.md @@ -0,0 +1,25 @@ +--- +slug: /en/sql-reference/table-functions/generate_series +sidebar_position: 146 +sidebar_label: generate_series +--- + +# generate_series + +`generate_series(START, STOP)` - Returns a table with the single ‘generate_series’ column (UInt64) that contains integers from start to stop inclusively. + +`generate_series(START, STOP, STEP)` - Returns a table with the single ‘generate_series’ column (UInt64) that contains integers from start to stop inclusively with spacing between values given by STEP. + +The following queries return tables with the same content but different column names: + +``` sql +SELECT * FROM numbers(10, 5); +SELECT * FROM generate_series(10, 14); +``` + +And the following queries return tables with the same content but different column names (but the second option is more efficient): + +``` sql +SELECT * FROM numbers(10, 11) WHERE number % 3 == (10 % 3); +SELECT * FROM generate_series(10, 20, 3) ; +``` \ No newline at end of file diff --git a/docs/en/sql-reference/table-functions/numbers.md b/docs/en/sql-reference/table-functions/numbers.md index 7d3437b7d53..e0ff19b9824 100644 --- a/docs/en/sql-reference/table-functions/numbers.md +++ b/docs/en/sql-reference/table-functions/numbers.md @@ -8,6 +8,7 @@ sidebar_label: numbers `numbers(N)` – Returns a table with the single ‘number’ column (UInt64) that contains integers from 0 to N-1. `numbers(N, M)` - Returns a table with the single ‘number’ column (UInt64) that contains integers from N to (N + M - 1). +`numbers(N, M, S)` - Returns a table with the single ‘number’ column (UInt64) that contains integers from N to (N + M - 1) with step S. Similar to the `system.numbers` table, it can be used for testing and generating successive values, `numbers(N, M)` more efficient than `system.numbers`. @@ -21,6 +22,15 @@ SELECT * FROM system.numbers WHERE number BETWEEN 0 AND 9; SELECT * FROM system.numbers WHERE number IN (0, 1, 2, 3, 4, 5, 6, 7, 8, 9); ``` +And the following queries are equivalent: + +``` sql +SELECT number * 2 FROM numbers(10); +SELECT (number - 10) * 2 FROM numbers(10, 10); +SELECT * FROM numbers(0, 20, 2); +``` + + Examples: ``` sql diff --git a/docs/en/sql-reference/transactions.md b/docs/en/sql-reference/transactions.md index cb89a091d68..09cdc192b03 100644 --- a/docs/en/sql-reference/transactions.md +++ b/docs/en/sql-reference/transactions.md @@ -127,7 +127,7 @@ See the [deployment](docs/en/deployment-guides/terminology.md) documentation for #### Verify that experimental transactions are enabled -Issue a `BEGIN TRANSACTION` followed by a `ROLLBACK` to verify that experimental transactions are enabled, and that ClickHouse Keeper is enabled as it is used to track transactions. +Issue a `BEGIN TRANSACTION` or `START TRANSACTION` followed by a `ROLLBACK` to verify that experimental transactions are enabled, and that ClickHouse Keeper is enabled as it is used to track transactions. ```sql BEGIN TRANSACTION diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md index 9b2ded7b6ce..32ebc6d028f 100644 --- a/docs/en/sql-reference/window-functions/index.md +++ b/docs/en/sql-reference/window-functions/index.md @@ -12,25 +12,23 @@ Some of the calculations that you can do are similar to those that can be done w ClickHouse supports the standard grammar for defining windows and window functions. The table below indicates whether a feature is currently supported. -| Feature | Support or workaround | +| Feature | Supported? | |------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | supported | -| expressions involving window functions, e.g. `(count(*) over ()) / 2)` | supported | -| `WINDOW` clause (`select ... from table window w as (partition by id)`) | supported | -| `ROWS` frame | supported | -| `RANGE` frame | supported, the default | -| `INTERVAL` syntax for `DateTime` `RANGE OFFSET` frame | not supported, specify the number of seconds instead (`RANGE` works with any numeric type). | -| `GROUPS` frame | not supported | -| Calculating aggregate functions over a frame (`sum(value) over (order by time)`) | all aggregate functions are supported | -| `rank()`, `dense_rank()`, `row_number()` | supported | -| `lag/lead(value, offset)` | Not supported. Workarounds: | -| | 1) replace with `any(value) over (.... rows between preceding and preceding)`, or `following` for `lead` | -| | 2) use `lagInFrame/leadInFrame`, which are analogous, but respect the window frame. To get behavior identical to `lag/lead`, use `rows between unbounded preceding and unbounded following` | -| ntile(buckets) | Supported. Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). | +| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | ✅ | +| expressions involving window functions, e.g. `(count(*) over ()) / 2)` | ✅ | +| `WINDOW` clause (`select ... from table window w as (partition by id)`) | ✅ | +| `ROWS` frame | ✅ | +| `RANGE` frame | ✅ (the default) | +| `INTERVAL` syntax for `DateTime` `RANGE OFFSET` frame | ❌ (specify the number of seconds instead (`RANGE` works with any numeric type).) | +| `GROUPS` frame | ❌ | +| Calculating aggregate functions over a frame (`sum(value) over (order by time)`) | ✅ (All aggregate functions are supported) | +| `rank()`, `dense_rank()`, `row_number()` | ✅ | +| `lag/lead(value, offset)` | ❌
You can use one of the following workarounds:
1) `any(value) over (.... rows between preceding and preceding)`, or `following` for `lead`
2) `lagInFrame/leadInFrame`, which are analogous, but respect the window frame. To get behavior identical to `lag/lead`, use `rows between unbounded preceding and unbounded following` | +| ntile(buckets) | ✅
Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). | ## ClickHouse-specific Window Functions -There are also the following window function that's specific to ClickHouse: +There is also the following ClickHouse specific window function: ### nonNegativeDerivative(metric_column, timestamp_column[, INTERVAL X UNITS]) @@ -89,6 +87,102 @@ These functions can be used only as a window function. Let's have a look at some examples of how window functions can be used. +### Numbering rows + +```sql +CREATE TABLE salaries +( + `team` String, + `player` String, + `salary` UInt32, + `position` String +) +Engine = Memory; + +INSERT INTO salaries FORMAT Values + ('Port Elizabeth Barbarians', 'Gary Chen', 195000, 'F'), + ('New Coreystad Archdukes', 'Charles Juarez', 190000, 'F'), + ('Port Elizabeth Barbarians', 'Michael Stanley', 150000, 'D'), + ('New Coreystad Archdukes', 'Scott Harrison', 150000, 'D'), + ('Port Elizabeth Barbarians', 'Robert George', 195000, 'M'); +``` + +```sql +SELECT player, salary, + row_number() OVER (ORDER BY salary) AS row +FROM salaries; +``` + +```text +┌─player──────────┬─salary─┬─row─┐ +│ Michael Stanley │ 150000 │ 1 │ +│ Scott Harrison │ 150000 │ 2 │ +│ Charles Juarez │ 190000 │ 3 │ +│ Gary Chen │ 195000 │ 4 │ +│ Robert George │ 195000 │ 5 │ +└─────────────────┴────────┴─────┘ +``` + +```sql +SELECT player, salary, + row_number() OVER (ORDER BY salary) AS row, + rank() OVER (ORDER BY salary) AS rank, + dense_rank() OVER (ORDER BY salary) AS denseRank +FROM salaries; +``` + +```text +┌─player──────────┬─salary─┬─row─┬─rank─┬─denseRank─┐ +│ Michael Stanley │ 150000 │ 1 │ 1 │ 1 │ +│ Scott Harrison │ 150000 │ 2 │ 1 │ 1 │ +│ Charles Juarez │ 190000 │ 3 │ 3 │ 2 │ +│ Gary Chen │ 195000 │ 4 │ 4 │ 3 │ +│ Robert George │ 195000 │ 5 │ 4 │ 3 │ +└─────────────────┴────────┴─────┴──────┴───────────┘ +``` + +### Aggregation functions + +Compare each player's salary to the average for their team. + +```sql +SELECT player, salary, team, + avg(salary) OVER (PARTITION BY team) AS teamAvg, + salary - teamAvg AS diff +FROM salaries; +``` + +```text +┌─player──────────┬─salary─┬─team──────────────────────┬─teamAvg─┬───diff─┐ +│ Charles Juarez │ 190000 │ New Coreystad Archdukes │ 170000 │ 20000 │ +│ Scott Harrison │ 150000 │ New Coreystad Archdukes │ 170000 │ -20000 │ +│ Gary Chen │ 195000 │ Port Elizabeth Barbarians │ 180000 │ 15000 │ +│ Michael Stanley │ 150000 │ Port Elizabeth Barbarians │ 180000 │ -30000 │ +│ Robert George │ 195000 │ Port Elizabeth Barbarians │ 180000 │ 15000 │ +└─────────────────┴────────┴───────────────────────────┴─────────┴────────┘ +``` + +Compare each player's salary to the maximum for their team. + +```sql +SELECT player, salary, team, + max(salary) OVER (PARTITION BY team) AS teamAvg, + salary - teamAvg AS diff +FROM salaries; +``` + +```text +┌─player──────────┬─salary─┬─team──────────────────────┬─teamAvg─┬───diff─┐ +│ Charles Juarez │ 190000 │ New Coreystad Archdukes │ 190000 │ 0 │ +│ Scott Harrison │ 150000 │ New Coreystad Archdukes │ 190000 │ -40000 │ +│ Gary Chen │ 195000 │ Port Elizabeth Barbarians │ 195000 │ 0 │ +│ Michael Stanley │ 150000 │ Port Elizabeth Barbarians │ 195000 │ -45000 │ +│ Robert George │ 195000 │ Port Elizabeth Barbarians │ 195000 │ 0 │ +└─────────────────┴────────┴───────────────────────────┴─────────┴────────┘ +``` + +### Partitioning by column + ```sql CREATE TABLE wf_partition ( @@ -120,6 +214,8 @@ ORDER BY └──────────┴───────┴───────┴──────────────┘ ``` +### Frame bounding + ```sql CREATE TABLE wf_frame ( @@ -131,14 +227,19 @@ ENGINE = Memory; INSERT INTO wf_frame FORMAT Values (1,1,1), (1,2,2), (1,3,3), (1,4,4), (1,5,5); +``` --- frame is bounded by bounds of a partition (BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) +```sql +-- Frame is bounded by bounds of a partition (BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) SELECT part_key, value, order, - groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC - Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS frame_values + groupArray(value) OVER ( + PARTITION BY part_key + ORDER BY order ASC + Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS frame_values FROM wf_frame ORDER BY part_key ASC, @@ -151,7 +252,9 @@ ORDER BY │ 1 │ 4 │ 4 │ [1,2,3,4,5] │ │ 1 │ 5 │ 5 │ [1,2,3,4,5] │ └──────────┴───────┴───────┴──────────────┘ +``` +```sql -- short form - no bound expression, no order by SELECT part_key, @@ -169,14 +272,19 @@ ORDER BY │ 1 │ 4 │ 4 │ [1,2,3,4,5] │ │ 1 │ 5 │ 5 │ [1,2,3,4,5] │ └──────────┴───────┴───────┴──────────────┘ +``` --- frame is bounded by the beggining of a partition and the current row +```sql +-- frame is bounded by the beginning of a partition and the current row SELECT part_key, value, order, - groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC - Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS frame_values + groupArray(value) OVER ( + PARTITION BY part_key + ORDER BY order ASC + Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS frame_values FROM wf_frame ORDER BY part_key ASC, @@ -189,8 +297,10 @@ ORDER BY │ 1 │ 4 │ 4 │ [1,2,3,4] │ │ 1 │ 5 │ 5 │ [1,2,3,4,5] │ └──────────┴───────┴───────┴──────────────┘ +``` --- short form (frame is bounded by the beggining of a partition and the current row) +```sql +-- short form (frame is bounded by the beginning of a partition and the current row) SELECT part_key, value, @@ -207,8 +317,10 @@ ORDER BY │ 1 │ 4 │ 4 │ [1,2,3,4] │ │ 1 │ 5 │ 5 │ [1,2,3,4,5] │ └──────────┴───────┴───────┴──────────────┘ +``` --- frame is bounded by the beggining of a partition and the current row, but order is backward +```sql +-- frame is bounded by the beginning of a partition and the current row, but order is backward SELECT part_key, value, @@ -225,14 +337,19 @@ ORDER BY │ 1 │ 4 │ 4 │ [5,4] │ │ 1 │ 5 │ 5 │ [5] │ └──────────┴───────┴───────┴──────────────┘ +``` +```sql -- sliding frame - 1 PRECEDING ROW AND CURRENT ROW SELECT part_key, value, order, - groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC - Rows BETWEEN 1 PRECEDING AND CURRENT ROW) AS frame_values + groupArray(value) OVER ( + PARTITION BY part_key + ORDER BY order ASC + Rows BETWEEN 1 PRECEDING AND CURRENT ROW + ) AS frame_values FROM wf_frame ORDER BY part_key ASC, @@ -245,14 +362,19 @@ ORDER BY │ 1 │ 4 │ 4 │ [3,4] │ │ 1 │ 5 │ 5 │ [4,5] │ └──────────┴───────┴───────┴──────────────┘ +``` +```sql -- sliding frame - Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING SELECT part_key, value, order, - groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC - Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS frame_values + groupArray(value) OVER ( + PARTITION BY part_key + ORDER BY order ASC + Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING + ) AS frame_values FROM wf_frame ORDER BY part_key ASC, @@ -264,7 +386,9 @@ ORDER BY │ 1 │ 4 │ 4 │ [3,4,5] │ │ 1 │ 5 │ 5 │ [4,5] │ └──────────┴───────┴───────┴──────────────┘ +``` +```sql -- row_number does not respect the frame, so rn_1 = rn_2 = rn_3 != rn_4 SELECT part_key, @@ -278,8 +402,11 @@ SELECT FROM wf_frame WINDOW w1 AS (PARTITION BY part_key ORDER BY order DESC), - w2 AS (PARTITION BY part_key ORDER BY order DESC - Rows BETWEEN 1 PRECEDING AND CURRENT ROW) + w2 AS ( + PARTITION BY part_key + ORDER BY order DESC + Rows BETWEEN 1 PRECEDING AND CURRENT ROW + ) ORDER BY part_key ASC, value ASC; @@ -290,7 +417,9 @@ ORDER BY │ 1 │ 4 │ 4 │ [5,4] │ 2 │ 2 │ 2 │ 2 │ │ 1 │ 5 │ 5 │ [5] │ 1 │ 1 │ 1 │ 1 │ └──────────┴───────┴───────┴──────────────┴──────┴──────┴──────┴──────┘ +``` +```sql -- first_value and last_value respect the frame SELECT groupArray(value) OVER w1 AS frame_values_1, @@ -313,7 +442,9 @@ ORDER BY │ [1,2,3,4] │ 1 │ 4 │ [3,4] │ 3 │ 4 │ │ [1,2,3,4,5] │ 1 │ 5 │ [4,5] │ 4 │ 5 │ └────────────────┴───────────────┴──────────────┴────────────────┴───────────────┴──────────────┘ +``` +```sql -- second value within the frame SELECT groupArray(value) OVER w1 AS frame_values_1, @@ -330,7 +461,9 @@ ORDER BY │ [1,2,3,4] │ 2 │ │ [2,3,4,5] │ 3 │ └────────────────┴──────────────┘ +``` +```sql -- second value within the frame + Null for missing values SELECT groupArray(value) OVER w1 AS frame_values_1, @@ -351,7 +484,9 @@ ORDER BY ## Real world examples -### Maximum/total salary per department. +The following examples solve common real-world problems. + +### Maximum/total salary per department ```sql CREATE TABLE employees @@ -369,7 +504,9 @@ INSERT INTO employees FORMAT Values ('IT', 'Tim', 200), ('IT', 'Anna', 300), ('IT', 'Elen', 500); +``` +```sql SELECT department, employee_name AS emp, @@ -386,8 +523,10 @@ FROM max(salary) OVER wndw AS max_salary_per_dep, sum(salary) OVER wndw AS total_salary_per_dep FROM employees - WINDOW wndw AS (PARTITION BY department - rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) + WINDOW wndw AS ( + PARTITION BY department + rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) ORDER BY department ASC, employee_name ASC @@ -403,7 +542,7 @@ FROM └────────────┴──────┴────────┴────────────────────┴──────────────────────┴──────────────────┘ ``` -### Cumulative sum. +### Cumulative sum ```sql CREATE TABLE warehouse @@ -421,7 +560,9 @@ INSERT INTO warehouse VALUES ('sku1', '2020-01-01', 1), ('sku1', '2020-02-01', 1), ('sku1', '2020-03-01', 1); +``` +```sql SELECT item, ts, @@ -461,13 +602,18 @@ insert into sensors values('cpu_temp', '2020-01-01 00:00:00', 87), ('cpu_temp', '2020-01-01 00:00:05', 87), ('cpu_temp', '2020-01-01 00:00:06', 87), ('cpu_temp', '2020-01-01 00:00:07', 87); +``` + +```sql SELECT metric, ts, value, - avg(value) OVER - (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN 2 PRECEDING AND CURRENT ROW) - AS moving_avg_temp + avg(value) OVER ( + PARTITION BY metric + ORDER BY ts ASC + Rows BETWEEN 2 PRECEDING AND CURRENT ROW + ) AS moving_avg_temp FROM sensors ORDER BY metric ASC, @@ -536,7 +682,9 @@ insert into sensors values('ambient_temp', '2020-01-01 00:00:00', 16), ('ambient_temp', '2020-03-01 12:00:00', 16), ('ambient_temp', '2020-03-01 12:00:00', 16), ('ambient_temp', '2020-03-01 12:00:00', 16); +``` +```sql SELECT metric, ts, diff --git a/docs/ru/index.md b/docs/ru/index.md index 78bb382753b..29f2bbe07fb 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -35,13 +35,13 @@ ClickHouse — столбцовая система управления база В примерах изображён только порядок расположения данных. То есть значения из разных столбцов хранятся отдельно, а данные одного столбца — вместе. -Примеры столбцовых СУБД: Vertica, Paraccel (Actian Matrix, Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise, Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid, kdb+. +Примеры столбцовых СУБД: Vertica, Paraccel (Actian Matrix, Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise, Actian Vector), LucidDB, SAP HANA и прочий треш, Google Dremel, Google PowerDrill, Druid, kdb+. {: .grey } Разный порядок хранения данных лучше подходит для разных сценариев работы. Сценарий работы с данными — это то, какие производятся запросы, как часто и в каком соотношении; сколько читается данных на запросы каждого вида — строк, столбцов, байтов; как соотносятся чтения и обновления данных; какой рабочий размер данных и насколько локально он используется; используются ли транзакции и с какой изолированностью; какие требования к дублированию данных и логической целостности; требования к задержкам на выполнение и пропускной способности запросов каждого вида и т. п. -Чем больше нагрузка на систему, тем более важной становится специализация под сценарий работы, и тем более конкретной становится эта специализация. Не существует системы, одинаково хорошо подходящей под существенно различные сценарии работы. Если система подходит под широкое множество сценариев работы, то при достаточно большой нагрузке, система будет справляться со всеми сценариями работы плохо, или справляться хорошо только с одним из сценариев работы. +Чем больше нагрузка на систему, тем более важной становится специализация под сценарий работы, и тем более конкретной становится эта специализация. Не существует системы, одинаково хорошо подходящей под существенно различные сценарии работы. Если система подходит под широкое множество сценариев работы, то при достаточно большой нагрузке система будет справляться со всеми сценариями работы плохо, или справляться хорошо только с одним из сценариев работы. ## Ключевые особенности OLAP-сценария работы {#kliuchevye-osobennosti-olap-stsenariia-raboty} @@ -53,11 +53,11 @@ ClickHouse — столбцовая система управления база - запросы идут сравнительно редко (обычно не более сотни в секунду на сервер); - при выполнении простых запросов, допустимы задержки в районе 50 мс; - значения в столбцах достаточно мелкие — числа и небольшие строки (например, 60 байт на URL); -- требуется высокая пропускная способность при обработке одного запроса (до миллиардов строк в секунду на один узел); +- требуется высокая пропускная способность при обработке одного запроса (до миллиардов строк в секунду на один сервер); - транзакции отсутствуют; -- низкие требования к консистентности данных; -- в запросе одна большая таблица, все таблицы кроме одной маленькие; -- результат выполнения запроса существенно меньше исходных данных — то есть данные фильтруются или агрегируются; результат выполнения помещается в оперативную память одного узла. +- низкие требования к согласованности данных; +- в запросе одна большая таблица, все остальные таблицы из запроса — маленькие; +- результат выполнения запроса существенно меньше исходных данных — то есть данные фильтруются или агрегируются; результат выполнения помещается в оперативную память одного сервера. Легко видеть, что OLAP-сценарий работы существенно отличается от других распространённых сценариев работы (например, OLTP или Key-Value сценариев работы). Таким образом, не имеет никакого смысла пытаться использовать OLTP-системы или системы класса «ключ — значение» для обработки аналитических запросов, если вы хотите получить приличную производительность («выше плинтуса»). Например, если вы попытаетесь использовать для аналитики MongoDB или Redis — вы получите анекдотически низкую производительность по сравнению с OLAP-СУБД. @@ -77,11 +77,11 @@ ClickHouse — столбцовая система управления база ### По вводу-выводу {#po-vvodu-vyvodu} -1. Для выполнения аналитического запроса, требуется прочитать небольшое количество столбцов таблицы. В столбцовой БД для этого можно читать только нужные данные. Например, если вам требуется только 5 столбцов из 100, то следует рассчитывать на 20-кратное уменьшение ввода-вывода. -2. Так как данные читаются пачками, то их проще сжимать. Данные, лежащие по столбцам также лучше сжимаются. За счёт этого, дополнительно уменьшается объём ввода-вывода. -3. За счёт уменьшения ввода-вывода, больше данных влезает в системный кэш. +1. Для выполнения аналитического запроса требуется прочитать небольшое количество столбцов таблицы. В столбцовой БД для этого можно читать только нужные данные. Например, если вам требуется только 5 столбцов из 100, то следует рассчитывать на 20-кратное уменьшение ввода-вывода. +2. Так как данные читаются пачками, то их проще сжимать. Данные, лежащие по столбцам, также лучше сжимаются. За счёт этого, дополнительно уменьшается объём ввода-вывода. +3. За счёт уменьшения ввода-вывода больше данных влезает в системный кэш. -Например, для запроса «посчитать количество записей для каждой рекламной системы», требуется прочитать один столбец «идентификатор рекламной системы», который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия, возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть, такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике, такая скорость действительно достигается. +Например, для запроса «посчитать количество записей для каждой рекламной системы» требуется прочитать один столбец «идентификатор рекламной системы», который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике такая скорость действительно достигается. ### По вычислениям {#po-vychisleniiam} @@ -96,4 +96,4 @@ ClickHouse — столбцовая система управления база В «обычных» СУБД этого не делается, так как не имеет смысла при выполнении простых запросов. Хотя есть исключения. Например, в MemSQL кодогенерация используется для уменьшения времени отклика при выполнении SQL-запросов. Для сравнения: в аналитических СУБД требуется оптимизация по пропускной способности (throughput, ГБ/с), а не времени отклика (latency, с). -Стоит заметить, что для эффективности по CPU требуется, чтобы язык запросов был декларативным (SQL, MDX) или хотя бы векторным (J, K). То есть необходимо, чтобы запрос содержал циклы только в неявном виде, открывая возможности для оптимизации. +Стоит заметить, что для эффективности по CPU требуется, чтобы язык запросов был декларативным (SQL, MDX) или хотя бы векторным (J, K, APL). То есть необходимо, чтобы запрос содержал циклы только в неявном виде, открывая возможности для оптимизации. diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index be8cfbdda6c..5f11f1b430b 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -434,16 +434,18 @@ $ curl -v 'http://localhost:8123/predefined_query' ``` xml - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> GET TEST_HEADER_VALUE - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> predefined_query_handler - SELECT value FROM system.settings WHERE name = {name_1:String} - SELECT name, value FROM system.settings WHERE name = {name_2:String} + + SELECT name, value FROM system.settings + WHERE name IN ({name_1:String}, {name_2:String}) + @@ -451,13 +453,13 @@ $ curl -v 'http://localhost:8123/predefined_query' ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/query_param_with_url/1/max_threads/max_final_threads?max_threads=1&max_final_threads=2' -1 -max_final_threads 2 +$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_final_threads' 'http://localhost:8123/query_param_with_url/max_threads?max_threads=1&max_final_threads=2' +max_final_threads 2 +max_threads 1 ``` :::note Предупреждение -В одном `predefined_query_handler` поддерживается только один запрос типа `INSERT`. +В одном `predefined_query_handler` поддерживается только один запрос. ::: ### dynamic_query_handler {#dynamic_query_handler} diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index a56afda641b..f9456e34a56 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2776,7 +2776,7 @@ SELECT range(number) FROM system.numbers LIMIT 5 FORMAT PrettyCompactNoEscapes; - 0 — номера строк не выводятся. - 1 — номера строк выводятся. -Значение по умолчанию: `0`. +Значение по умолчанию: `1`. **Пример** @@ -2798,7 +2798,7 @@ SELECT TOP 3 name, value FROM system.settings; ``` ### output_format_pretty_color {#output_format_pretty_color} -Включает/выключает управляющие последовательности ANSI в форматах Pretty. +Включает/выключает управляющие последовательности ANSI в форматах Pretty. Возможные значения: @@ -4123,7 +4123,7 @@ SELECT sum(number) FROM numbers(10000000000) SETTINGS partial_result_on_first_ca ## session_timezone {#session_timezone} Задаёт значение часового пояса (session_timezone) по умолчанию для текущей сессии вместо [часового пояса сервера](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone). То есть, все значения DateTime/DateTime64, для которых явно не задан часовой пояс, будут интерпретированы как относящиеся к указанной зоне. -При значении настройки `''` (пустая строка), будет совпадать с часовым поясом сервера. +При значении настройки `''` (пустая строка), будет совпадать с часовым поясом сервера. Функции `timeZone()` and `serverTimezone()` возвращают часовой пояс текущей сессии и сервера соответственно. diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index 59a9c7f8cf1..6463f6bd95d 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -476,7 +476,7 @@ FROM - `r1` - количество уникальных посетителей за 2020-01-01 (`cond1`). - `r2` - количество уникальных посетителей в период между 2020-01-01 и 2020-01-02 (`cond1` и `cond2`). -- `r3` - количество уникальных посетителей в период между 2020-01-01 и 2020-01-03 (`cond1` и `cond3`). +- `r3` - количество уникальных посетителей в период за 2020-01-01 и 2020-01-03 (`cond1` и `cond3`). ## uniqUpTo(N)(x) {#uniquptonx} diff --git a/docs/ru/sql-reference/data-types/datetime.md b/docs/ru/sql-reference/data-types/datetime.md index 57f24786bb7..34cd44d4709 100644 --- a/docs/ru/sql-reference/data-types/datetime.md +++ b/docs/ru/sql-reference/data-types/datetime.md @@ -120,7 +120,7 @@ FROM dt - [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md) - [Функции для работы с массивами](../../sql-reference/functions/array-functions.md) - [Настройка `date_time_input_format`](../../operations/settings/index.md#settings-date_time_input_format) -- [Настройка `date_time_output_format`](../../operations/settings/index.md) +- [Настройка `date_time_output_format`](../../operations/settings/index.md#settings-date_time_output_format) - [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) - [Параметр `session_timezone`](../../operations/settings/settings.md#session_timezone) - [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime) diff --git a/docs/ru/sql-reference/data-types/multiword-types.md b/docs/ru/sql-reference/data-types/multiword-types.md deleted file mode 100644 index cca2d71e480..00000000000 --- a/docs/ru/sql-reference/data-types/multiword-types.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -slug: /ru/sql-reference/data-types/multiword-types -sidebar_position: 61 -sidebar_label: Составные типы ---- - -# Составные типы {#multiword-types} - -При создании таблиц вы можете использовать типы данных с названием, состоящим из нескольких слов. Такие названия поддерживаются для лучшей совместимости с SQL. - -## Поддержка составных типов {#multiword-types-support} - -| Составные типы | Обычные типы | -|-------------------------------------|-----------------------------------------------------------| -| DOUBLE PRECISION | [Float64](../../sql-reference/data-types/float.md) | -| CHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| CHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | -| NCHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| NCHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHAR VARYING | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHARACTER | [String](../../sql-reference/data-types/string.md) | -| NATIONAL CHAR | [String](../../sql-reference/data-types/string.md) | -| BINARY LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | -| BINARY VARYING | [String](../../sql-reference/data-types/string.md) | diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index cbbb456aa80..56ae4359bf1 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -627,7 +627,7 @@ SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(d ## age -Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 микросекунду. +Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 наносекунду. Например, разница между `2021-12-29` и `2022-01-01` 3 дня для единицы `day`, 0 месяцев для единицы `month`, 0 лет для единицы `year`. **Синтаксис** @@ -641,6 +641,7 @@ age('unit', startdate, enddate, [timezone]) - `unit` — единица измерения времени, в которой будет выражено возвращаемое значение функции. [String](../../sql-reference/data-types/string.md). Возможные значения: + - `nanosecond` (возможные сокращения: `ns`) - `microsecond` (возможные сокращения: `us`, `u`) - `millisecond` (возможные сокращения: `ms`) - `second` (возможные сокращения: `ss`, `s`) @@ -716,6 +717,7 @@ date_diff('unit', startdate, enddate, [timezone]) - `unit` — единица измерения времени, в которой будет выражено возвращаемое значение функции. [String](../../sql-reference/data-types/string.md). Возможные значения: + - `nanosecond` (возможные сокращения: `ns`) - `microsecond` (возможные сокращения: `us`, `u`) - `millisecond` (возможные сокращения: `ms`) - `second` (возможные сокращения: `ss`, `s`) diff --git a/docs/zh/engines/database-engines/materialize-mysql.md b/docs/zh/engines/database-engines/materialize-mysql.md deleted file mode 100644 index 5d1394f9456..00000000000 --- a/docs/zh/engines/database-engines/materialize-mysql.md +++ /dev/null @@ -1,196 +0,0 @@ ---- -slug: /zh/engines/database-engines/materialize-mysql -sidebar_position: 29 -sidebar_label: "[experimental] MaterializedMySQL" ---- - -# [experimental] MaterializedMySQL {#materialized-mysql} - -**这是一个实验性的特性，不应该在生产中使用。** - -创建ClickHouse数据库，包含MySQL中所有的表，以及这些表中的所有数据。 - -ClickHouse服务器作为MySQL副本工作。它读取binlog并执行DDL和DML查询。 - -这个功能是实验性的。 - -## 创建数据库 {#creating-a-database} - -``` sql -CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MaterializeMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...] -``` - -**引擎参数** - -- `host:port` — MySQL服务地址 -- `database` — MySQL数据库名称 -- `user` — MySQL用户名 -- `password` — MySQL用户密码 - -**引擎配置** - -- `max_rows_in_buffer` — 允许数据缓存到内存中的最大行数(对于单个表和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `65505`。 -- `max_bytes_in_buffer` — 允许在内存中缓存数据的最大字节数(对于单个表和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `1048576`. -- `max_rows_in_buffers` — 允许数据缓存到内存中的最大行数(对于数据库和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `65505`. -- `max_bytes_in_buffers` — 允许在内存中缓存数据的最大字节数(对于数据库和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `1048576`. -- `max_flush_data_time` — 允许数据在内存中缓存的最大毫秒数(对于数据库和无法查询的缓存数据)。当超过这个时间时，数据将被物化。默认值: `1000`. -- `max_wait_time_when_mysql_unavailable` — 当MySQL不可用时重试间隔(毫秒)。负值禁止重试。默认值: `1000`. -- `allows_query_when_mysql_lost` — 当mysql丢失时，允许查询物化表。默认值: `0` (`false`). -``` -CREATE DATABASE mysql ENGINE = MaterializeMySQL('localhost:3306', 'db', 'user', '***') - SETTINGS - allows_query_when_mysql_lost=true, - max_wait_time_when_mysql_unavailable=10000; -``` - -**MySQL服务器端配置** - -为了`MaterializeMySQL`正确的工作，有一些强制性的`MySQL`侧配置设置应该设置: - -- `default_authentication_plugin = mysql_native_password`，因为`MaterializeMySQL`只能使用此方法授权。 -- `gtid_mode = on`，因为要提供正确的`MaterializeMySQL`复制，基于GTID的日志记录是必须的。注意，在打开这个模式`On`时，你还应该指定`enforce_gtid_consistency = on`。 - -## 虚拟列 {#virtual-columns} - -当使用`MaterializeMySQL`数据库引擎时，[ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md)表与虚拟的`_sign`和`_version`列一起使用。 - -- `_version` — 同步版本。 类型[UInt64](../../sql-reference/data-types/int-uint.md). -- `_sign` — 删除标记。类型 [Int8](../../sql-reference/data-types/int-uint.md). Possible values: - - `1` — 行不会删除, - - `-1` — 行被删除。 - -## 支持的数据类型 {#data_types-support} - -| MySQL | ClickHouse | -|-------------------------|--------------------------------------------------------------| -| TINY | [Int8](../../sql-reference/data-types/int-uint.md) | -| SHORT | [Int16](../../sql-reference/data-types/int-uint.md) | -| INT24 | [Int32](../../sql-reference/data-types/int-uint.md) | -| LONG | [UInt32](../../sql-reference/data-types/int-uint.md) | -| LONGLONG | [UInt64](../../sql-reference/data-types/int-uint.md) | -| FLOAT | [Float32](../../sql-reference/data-types/float.md) | -| DOUBLE | [Float64](../../sql-reference/data-types/float.md) | -| DECIMAL, NEWDECIMAL | [Decimal](../../sql-reference/data-types/decimal.md) | -| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) | -| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) | -| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) | -| ENUM | [Enum](../../sql-reference/data-types/enum.md) | -| STRING | [String](../../sql-reference/data-types/string.md) | -| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) | -| BLOB | [String](../../sql-reference/data-types/string.md) | -| BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) | - -不支持其他类型。如果MySQL表包含此类类型的列，ClickHouse抛出异常"Unhandled data type"并停止复制。 - -[Nullable](../../sql-reference/data-types/nullable.md)已经支持 - -## 使用方式 {#specifics-and-recommendations} - -### 兼容性限制 - -除了数据类型的限制外，与`MySQL`数据库相比，还存在一些限制，在实现复制之前应先解决这些限制： - -- `MySQL`中的每个表都应该包含`PRIMARY KEY` - -- 对于包含`ENUM`字段值超出范围（在`ENUM`签名中指定）的行的表，复制将不起作用。 - -### DDL查询 {#ddl-queries} - -MySQL DDL查询转换为相应的ClickHouse DDL查询([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md))。如果ClickHouse无法解析某个DDL查询，则该查询将被忽略。 - -### Data Replication {#data-replication} - -`MaterializeMySQL`不支持直接`INSERT`, `DELETE`和`UPDATE`查询. 但是，它们是在数据复制方面支持的： - -- MySQL的`INSERT`查询转换为`INSERT`并携带`_sign=1`. - -- MySQL的`DELETE`查询转换为`INSERT`并携带`_sign=-1`. - -- MySQL的`UPDATE`查询转换为`INSERT`并携带`_sign=-1`, `INSERT`和`_sign=1`. - -### 查询MaterializeMySQL表 {#select} - -`SELECT`查询`MaterializeMySQL`表有一些细节: - -- 如果`_version`在`SELECT`中没有指定，则使用[FINAL](../../sql-reference/statements/select/from.md#select-from-final)修饰符。所以只有带有`MAX(_version)`的行才会被选中。 - -- 如果`_sign`在`SELECT`中没有指定，则默认使用`WHERE _sign=1`。因此，删除的行不会包含在结果集中。 - -- 结果包括列中的列注释，因为它们存在于SQL数据库表中。 - -### Index Conversion {#index-conversion} - -MySQL的`PRIMARY KEY`和`INDEX`子句在ClickHouse表中转换为`ORDER BY`元组。 - -ClickHouse只有一个物理顺序，由`ORDER BY`子句决定。要创建一个新的物理顺序，使用[materialized views](../../sql-reference/statements/create/view.md#materialized)。 - -**Notes** - -- 带有`_sign=-1`的行不会从表中物理删除。 -- `MaterializeMySQL`引擎不支持级联`UPDATE/DELETE`查询。 -- 复制很容易被破坏。 -- 禁止对数据库和表进行手工操作。 -- `MaterializeMySQL`受[optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert)设置的影响。当MySQL服务器中的表发生变化时，数据会合并到`MaterializeMySQL`数据库中相应的表中。 - -## 使用示例 {#examples-of-use} - -MySQL操作: - -``` sql -mysql> CREATE DATABASE db; -mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT); -mysql> INSERT INTO db.test VALUES (1, 11), (2, 22); -mysql> DELETE FROM db.test WHERE a=1; -mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16); -mysql> UPDATE db.test SET c='Wow!', b=222; -mysql> SELECT * FROM test; -``` - -```text -+---+------+------+ -| a | b | c | -+---+------+------+ -| 2 | 222 | Wow! | -+---+------+------+ -``` - -ClickHouse中的数据库，与MySQL服务器交换数据: - -创建的数据库和表: - -``` sql -CREATE DATABASE mysql ENGINE = MaterializeMySQL('localhost:3306', 'db', 'user', '***'); -SHOW TABLES FROM mysql; -``` - -``` text -┌─name─┐ -│ test │ -└──────┘ -``` - -然后插入数据: - -``` sql -SELECT * FROM mysql.test; -``` - -``` text -┌─a─┬──b─┐ -│ 1 │ 11 │ -│ 2 │ 22 │ -└───┴────┘ -``` - -删除数据后，添加列并更新: - -``` sql -SELECT * FROM mysql.test; -``` - -``` text -┌─a─┬───b─┬─c────┐ -│ 2 │ 222 │ Wow! │ -└───┴─────┴──────┘ -``` diff --git a/docs/zh/interfaces/http.md b/docs/zh/interfaces/http.md index 84ca5ed0c47..f55cf41936f 100644 --- a/docs/zh/interfaces/http.md +++ b/docs/zh/interfaces/http.md @@ -427,29 +427,32 @@ $ curl -v 'http://localhost:8123/predefined_query' ``` xml - [^/]+)(/(?P[^/]+))?]]> - GET + [^/]+)]]> + GET TEST_HEADER_VALUE - [^/]+)(/(?P[^/]+))?]]> + [^/]+)]]> predefined_query_handler - SELECT value FROM system.settings WHERE name = {name_1:String} - SELECT name, value FROM system.settings WHERE name = {name_2:String} + + SELECT name, value FROM system.settings + WHERE name IN ({name_1:String}, {name_2:String}) + + ``` ``` bash -$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost:8123/query_param_with_url/1/max_threads/max_final_threads?max_threads=1&max_final_threads=2' -1 -max_final_threads 2 +$ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_final_threads' 'http://localhost:8123/query_param_with_url/max_threads?max_threads=1&max_final_threads=2' +max_final_threads 2 +max_threads 1 ``` :::warning -在一个`predefined_query_handler`中，只支持insert类型的一个`查询`。 +在一个`predefined_query_handler`中，只支持的一个`查询`。 ::: ### 动态查询 {#dynamic_query_handler} diff --git a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md index 1c7de515c58..cb1dcc35f5c 100644 --- a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md @@ -472,7 +472,7 @@ FROM - `r1`-2020-01-01期间访问该网站的独立访问者数量（ `cond1` 条件）。 - `r2`-在2020-01-01和2020-01-02之间的特定时间段内访问该网站的唯一访问者的数量 (`cond1` 和 `cond2` 条件）。 -- `r3`-在2020-01-01和2020-01-03之间的特定时间段内访问该网站的唯一访问者的数量 (`cond1` 和 `cond3` 条件）。 +- `r3`-在2020-01-01和2020-01-03 网站的独立访客数量 (`cond1` 和 `cond3` 条件）。 ## uniqUpTo(N)(x) {#uniquptonx} diff --git a/docs/zh/sql-reference/data-types/multiword-types.mdx b/docs/zh/sql-reference/data-types/multiword-types.mdx deleted file mode 100644 index 85431d47efd..00000000000 --- a/docs/zh/sql-reference/data-types/multiword-types.mdx +++ /dev/null @@ -1,10 +0,0 @@ ---- -slug: /zh/sql-reference/data-types/multiword-types -sidebar_position: 61 -sidebar_label: Multiword Type Names -title: "Multiword Types" ---- - -import Content from '@site/docs/en/sql-reference/data-types/multiword-types.md'; - - diff --git a/docs/zh/sql-reference/functions/date-time-functions.md b/docs/zh/sql-reference/functions/date-time-functions.md index e4b70322477..d6493ffe605 100644 --- a/docs/zh/sql-reference/functions/date-time-functions.md +++ b/docs/zh/sql-reference/functions/date-time-functions.md @@ -643,6 +643,7 @@ date_diff('unit', startdate, enddate, [timezone]) - `unit` — `value`对应的时间单位。类型为[String](../../sql-reference/data-types/string.md)。 可能的值： + - `nanosecond` - `microsecond` - `millisecond` - `second` diff --git a/docs/zh/sql-reference/functions/string-search-functions.md b/docs/zh/sql-reference/functions/string-search-functions.md index e4167127424..972fd84e2a1 100644 --- a/docs/zh/sql-reference/functions/string-search-functions.md +++ b/docs/zh/sql-reference/functions/string-search-functions.md @@ -1,128 +1,702 @@ --- slug: /zh/sql-reference/functions/string-search-functions --- -# 字符串搜索函数 {#zi-fu-chuan-sou-suo-han-shu} -下列所有函数在默认的情况下区分大小写。对于不区分大小写的搜索，存在单独的变体。 +# 字符串搜索函数 -## 位置（大海捞针），定位（大海捞针) {#positionhaystack-needle-locatehaystack-needle} +本节中的所有函数默认情况下都区分大小写进行搜索。不区分大小写的搜索通常由单独的函数变体提供。 +请注意，不区分大小写的搜索，遵循英语的小写-大写规则。 +例如。英语中大写的`i`是`I`，而在土耳其语中则是`İ`, 对于英语以外的语言，结果可能会不符合预期。 -在字符串`haystack`中搜索子串`needle`。 -返回子串的位置（以字节为单位），从1开始，如果未找到子串，则返回0。 +本节中的函数还假设搜索字符串和被搜索字符串是单字节编码文本(例如ASCII)。如果违反此假设，不会抛出异常且结果为undefined。 +UTF-8 编码字符串的搜索通常由单独的函数变体提供。同样，如果使用 UTF-8 函数变体但输入字符串不是 UTF-8 编码文本，不会抛出异常且结果为 undefined。 +需要注意，函数不会执行自动 Unicode 规范化，您可以使用[normalizeUTF8*()](https://clickhouse.com/docs/zh/sql-reference/functions/string-functions/) 函数来执行此操作。 +在[字符串函数](string-functions.md) 和 [字符串替换函数](string-replace-functions.md) 会分别说明. -对于不区分大小写的搜索，请使用函数`positionCaseInsensitive`。 +## position -## positionUTF8(大海捞针) {#positionutf8haystack-needle} +返回字符串`haystack`中子字符串`needle`的位置（以字节为单位，从 1 开始）。 -与`position`相同，但位置以Unicode字符返回。此函数工作在UTF-8编码的文本字符集中。如非此编码的字符集，则返回一些非预期结果（他不会抛出异常）。 +**语法** -对于不区分大小写的搜索，请使用函数`positionCaseInsensitiveUTF8`。 +``` sql +position(haystack, needle[, start_pos]) +``` -## 多搜索分配（干草堆，\[针1，针2, …, needlen\]) {#multisearchallpositionshaystack-needle1-needle2-needlen} +别名： +- `position(needle IN haystack)` -与`position`相同，但函数返回一个数组，其中包含所有匹配needle我的位置。 +**参数** -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchAllPositionsCaseInsensitive，multiSearchAllPositionsUTF8，multiSearchAllPositionsCaseInsensitiveUTF8`。 +- `haystack` — 被检索查询字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 进行查询的子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` – 在字符串`haystack` 中开始检索的位置(从1开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选。 -## multiSearchFirstPosition(大海捞针,\[针1，针2, …, needlen\]) {#multisearchfirstpositionhaystack-needle1-needle2-needlen} +**返回值** -与`position`相同，但返回在`haystack`中与needles字符串匹配的最左偏移。 +- 若子字符串存在，返回位置(以字节为单位，从 1 开始）。 +- 如果不存在子字符串，返回 0。 -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchFirstPositionCaseInsensitive，multiSearchFirstPositionUTF8，multiSearchFirstPositionCaseInsensitiveUTF8`。 +如果子字符串 `needle` 为空，则： +- 如果未指定 `start_pos`，返回 `1` +- 如果 `start_pos = 0`，则返回 `1` +- 如果 `start_pos >= 1` 且 `start_pos <= length(haystack) + 1`，则返回 `start_pos` +- 否则返回 `0` -## multiSearchFirstIndex(大海捞针,\[针1，针2, …, needlen\]) {#multisearchfirstindexhaystack-needle1-needle2-needlen} +以上规则同样在这些函数中生效: [locate](#locate), [positionCaseInsensitive](#positionCaseInsensitive), [positionUTF8](#positionUTF8), [positionCaseInsensitiveUTF8](#positionCaseInsensitiveUTF8) -返回在字符串`haystack`中最先查找到的needle我的索引`i`（从1开始），没有找到任何匹配项则返回0。 +数据类型: `Integer`. -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchFirstIndexCaseInsensitive，multiSearchFirstIndexUTF8，multiSearchFirstIndexCaseInsensitiveUTF8`。 +**示例** -## 多搜索（大海捞针，\[针1，针2, …, needlen\]) {#multisearchanyhaystack-needle1-needle2-needlen} +``` sql +SELECT position('Hello, world!', '!'); +``` -如果`haystack`中至少存在一个needle我匹配则返回1，否则返回0。 +结果： + +``` text +┌─position('Hello, world!', '!')─┐ +│ 13 │ +└────────────────────────────────┘ +``` + +示例，使用参数 `start_pos` : + +``` sql +SELECT + position('Hello, world!', 'o', 1), + position('Hello, world!', 'o', 7) +``` + +结果： + +``` text +┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐ +│ 5 │ 9 │ +└───────────────────────────────────┴───────────────────────────────────┘ +``` + +示例，`needle IN haystack`: + +```sql +SELECT 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); +``` + +结果： + +```text +┌─equals(6, position(s, '/'))─┐ +│ 1 │ +└─────────────────────────────┘ +``` + +示例，子字符串 `needle` 为空: + +``` sql +SELECT + position('abc', ''), + position('abc', '', 0), + position('abc', '', 1), + position('abc', '', 2), + position('abc', '', 3), + position('abc', '', 4), + position('abc', '', 5) +``` +结果： +``` text +┌─position('abc', '')─┬─position('abc', '', 0)─┬─position('abc', '', 1)─┬─position('abc', '', 2)─┬─position('abc', '', 3)─┬─position('abc', '', 4)─┬─position('abc', '', 5)─┐ +│ 1 │ 1 │ 1 │ 2 │ 3 │ 4 │ 0 │ +└─────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┘ +``` + +## locate + +类似于 [position](#position) 但交换了 `haystack` 和 `locate` 参数。 + +此函数的行为取决于 ClickHouse 版本： +- 在 v24.3 以下的版本中，`locate` 是函数`position`的别名，参数为 `(haystack, needle[, start_pos])`。 +- 在 v24.3 及以上的版本中,, `locate` 是独立的函数 (以更好地兼容 MySQL) ,参数为 `(needle, haystack[, start_pos])`。 之前的行为 + 可以在设置中恢复 [function_locate_has_mysql_compatible_argument_order = false](../../operations/settings/settings.md#function-locate-has-mysql-compatible-argument-order); + +**语法** + +``` sql +locate(needle, haystack[, start_pos]) +``` + +## positionCaseInsensitive + +类似于 [position](#position) 但是不区分大小写。 + +## positionUTF8 + +类似于 [position](#position) 但是假定 `haystack` 和 `needle` 是 UTF-8 编码的字符串。 + +**示例** + +函数 `positionUTF8` 可以正确的将字符 `ö` 计为单个 Unicode 代码点(`ö`由两个点表示): + +``` sql +SELECT positionUTF8('Motörhead', 'r'); +``` + +结果： + +``` text +┌─position('Motörhead', 'r')─┐ +│ 5 │ +└────────────────────────────┘ +``` + +## positionCaseInsensitiveUTF8 + +类似于 [positionUTF8](#positionutf8) 但是不区分大小写。 + +## multiSearchAllPositions + +类似于 [position](#position) 但是返回多个在字符串 `haystack` 中 `needle` 子字符串的位置的数组（以字节为单位，从 1 开始）。 -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchAnyCaseInsensitive，multiSearchAnyUTF8，multiSearchAnyCaseInsensitiveUTF8`。 :::note -在所有`multiSearch*`函数中，由于实现规范，needles的数量应小于28。 +所有以 `multiSearch*()` 开头的函数仅支持最多 28 个`needle`. ::: -## 匹配（大海捞针，模式) {#matchhaystack-pattern} +**语法** -检查字符串是否与`pattern`正则表达式匹配。`pattern`可以是一个任意的`re2`正则表达式。 `re2`正则表达式的[语法](https://github.com/google/re2/wiki/Syntax)比Perl正则表达式的语法存在更多限制。 +``` sql +multiSearchAllPositions(haystack, [needle1, needle2, ..., needleN]) +``` -如果不匹配返回0，否则返回1。 +**参数** -请注意，反斜杠符号（`\`）用于在正则表达式中转义。由于字符串中采用相同的符号来进行转义。因此，为了在正则表达式中转义符号，必须在字符串文字中写入两个反斜杠（\\）。 +- `haystack` — 被检索查询字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 子字符串数组, 类型为[Array](../../sql-reference/data-types/array.md) -正则表达式与字符串一起使用，就像它是一组字节一样。正则表达式中不能包含空字节。 -对于在字符串中搜索子字符串的模式，最好使用LIKE或«position»，因为它们更加高效。 +**返回值** -## multiMatchAny（大海捞针，\[模式1，模式2, …, patternn\]) {#multimatchanyhaystack-pattern1-pattern2-patternn} +- 位置数组，数组中的每个元素对应于 `needle` 数组中的一个元素。如果在 `haystack` 中找到子字符串，则返回的数组中的元素为子字符串的位置（以字节为单位，从 1 开始）；如果未找到子字符串，则返回的数组中的元素为 0。 -与`match`相同，但如果所有正则表达式都不匹配，则返回0；如果任何模式匹配，则返回1。它使用[超扫描](https://github.com/intel/hyperscan)库。对于在字符串中搜索子字符串的模式，最好使用«multisearchany»，因为它更高效。 +**示例** + +``` sql +SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']); +``` + +结果： + +``` text +┌─multiSearchAllPositions('Hello, World!', ['hello', '!', 'world'])─┐ +│ [0,13,0] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +## multiSearchAllPositionsUTF8 + +类似于 [multiSearchAllPositions](#multiSearchAllPositions) ,但假定 `haystack` 和 `needle`-s 是 UTF-8 编码的字符串。 + +## multiSearchFirstPosition + +类似于 `position` , 在字符串`haystack`中匹配多个`needle`子字符串，从左开始任一匹配的子串，返回其位置。 + +函数 `multiSearchFirstPositionCaseInsensitive`, `multiSearchFirstPositionUTF8` 和 `multiSearchFirstPositionCaseInsensitiveUTF8` 提供此函数的不区分大小写 以及/或 UTF-8 变体。 + +**语法** + +```sql +multiSearchFirstPosition(haystack, [needle1, needle2, …, needleN]) +``` + +## multiSearchFirstIndex + +在字符串`haystack`中匹配最左侧的 needlei 子字符串，返回其索引 `i` (从1开始)，如无法匹配则返回0。 + +函数 `multiSearchFirstIndexCaseInsensitive`, `multiSearchFirstIndexUTF8` 和 `multiSearchFirstIndexCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + +**语法** + +```sql +multiSearchFirstIndex(haystack, \[needle1, needle2, …, needlen\]) +``` + +## multiSearchAny {#multisearchany} + +至少已有一个子字符串`needle`匹配 `haystack` 时返回1，否则返回 0 。 + +函数 `multiSearchAnyCaseInsensitive`, `multiSearchAnyUTF8` 和 `multiSearchAnyCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + + +**语法** + +```sql +multiSearchAny(haystack, [needle1, needle2, …, needleN]) +``` + +## match {#match} + +返回字符串 `haystack` 是否匹配正则表达式 `pattern` （[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) + +匹配基于 UTF-8，例如`.` 匹配 Unicode 代码点 `¥`，它使用两个字节以 UTF-8 表示。T正则表达式不得包含空字节。如果 `haystack` 或`pattern`不是有效的 UTF-8，则此行为为undefined。 +与 re2 的默认行为不同，`.` 会匹配换行符。要禁用此功能，请在模式前面添加`(?-s)`。 + +如果仅希望搜索子字符串，可以使用函数 [like](#like)或 [position](#position) 来替代，这些函数的性能比此函数更高。 + +**语法** + +```sql +match(haystack, pattern) +``` + +别名： `haystack REGEXP pattern operator` + +## multiMatchAny + +类似于 `match`，如果至少有一个表达式 `patterni` 匹配字符串 `haystack`，则返回1，否则返回0。 :::note -任何`haystack`字符串的长度必须小于232\字节，否则抛出异常。这种限制是因为hyperscan API而产生的。 +`multi[Fuzzy]Match*()` 函数家族使用了(Vectorscan)[https://github.com/VectorCamp/vectorscan]库. 因此，只有当 ClickHouse 编译时支持矢量扫描时，它们才会启用。 + +要关闭所有使用矢量扫描(hyperscan)的功能，请使用设置 `SET allow_hyperscan = 0;`。 + +由于Vectorscan的限制，`haystack` 字符串的长度必须小于232字节。 + +Hyperscan 通常容易受到正则表达式拒绝服务 (ReDoS) 攻击。有关更多信息，请参见 +[https://www.usenix.org/conference/usenixsecurity22/presentation/turonova](https://www.usenix.org/conference/usenixsecurity22/presentation/turonova) +[https://doi.org/10.1007/s10664-021-10033-1](https://doi.org/10.1007/s10664-021-10033-1) +[https://doi.org/10.1145/3236024.3236027](https://doi.org/10.1145/3236024.3236027) +建议用户谨慎检查提供的表达式。 + ::: -## multiMatchAnyIndex（大海捞针，\[模式1，模式2, …, patternn\]) {#multimatchanyindexhaystack-pattern1-pattern2-patternn} +如果仅希望搜索子字符串，可以使用函数 [multiSearchAny](#multisearchany) 来替代，这些函数的性能比此函数更高。 -与`multiMatchAny`相同，但返回与haystack匹配的任何内容的索引位置。 +**语法** -## multiFuzzyMatchAny(干草堆,距离,\[模式1，模式2, …, patternn\]) {#multifuzzymatchanyhaystack-distance-pattern1-pattern2-patternn} +```sql +multiMatchAny(haystack, \[pattern1, pattern2, …, patternn\]) +``` -与`multiMatchAny`相同，但如果在haystack能够查找到任何模式匹配能够在指定的[编辑距离](https://en.wikipedia.org/wiki/Edit_distance)内进行匹配，则返回1。此功能也处于实验模式，可能非常慢。有关更多信息，请参阅[hyperscan文档](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching)。 +## multiMatchAnyIndex -## multiFuzzyMatchAnyIndex(大海捞针,距离,\[模式1，模式2, …, patternn\]) {#multifuzzymatchanyindexhaystack-distance-pattern1-pattern2-patternn} +类似于 `multiMatchAny` ，返回任何子串匹配 `haystack` 的索引。 -与`multiFuzzyMatchAny`相同，但返回匹配项的匹配能容的索引位置。 +**语法** + +```sql +multiMatchAnyIndex(haystack, \[pattern1, pattern2, …, patternn\]) +``` + +## multiMatchAllIndices + +类似于 `multiMatchAny`，返回一个数组，包含所有匹配 `haystack` 的子字符串的索引。 + +**语法** + +```sql +multiMatchAllIndices(haystack, \[pattern1, pattern2, …, patternn\]) +``` + +## multiFuzzyMatchAny + +类似于 `multiMatchAny` ，如果任一 `pattern` 匹配 `haystack`，则返回1 within a constant [edit distance](https://en.wikipedia.org/wiki/Edit_distance). 该功能依赖于实验特征 [hyperscan](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching) 库，并且对于某些边缘场景可能会很慢。性能取决于编辑距离`distance`的值和使用的`partten`，但与非模糊搜索相比，它的开销总是更高的。 :::note -`multiFuzzyMatch*`函数不支持UTF-8正则表达式，由于hyperscan限制，这些表达式被按字节解析。 +由于 hyperscan 的限制，`multiFuzzyMatch*()` 函数族不支持 UTF-8 正则表达式（hyperscan以一串字节来处理）。 ::: +**语法** + +```sql +multiFuzzyMatchAny(haystack, distance, \[pattern1, pattern2, …, patternn\]) +``` + +## multiFuzzyMatchAnyIndex + +类似于 `multiFuzzyMatchAny` 返回在编辑距离内与`haystack`匹配的任何索引 + +**语法** + +```sql +multiFuzzyMatchAnyIndex(haystack, distance, \[pattern1, pattern2, …, patternn\]) +``` + +## multiFuzzyMatchAllIndices + +类似于 `multiFuzzyMatchAny` 返回在编辑距离内与`haystack`匹配的所有索引的数组。 + +**语法** + +```sql +multiFuzzyMatchAllIndices(haystack, distance, \[pattern1, pattern2, …, patternn\]) +``` + +## extract + +使用正则表达式提取字符串。如果字符串 `haystack` 不匹配正则表达式 `pattern` ，则返回空字符串。 + +对于没有子模式的正则表达式，该函数使用与整个正则表达式匹配的片段。否则，它使用与第一个子模式匹配的片段。 + +**语法** + +```sql +extract(haystack, pattern) +``` + +## extractAll + +使用正则表达式提取字符串的所有片段。如果字符串 `haystack` 不匹配正则表达式 `pattern` ，则返回空字符串。 + +返回所有匹配项组成的字符串数组。 + +子模式的行为与函数`extract`中的行为相同。 + +**语法** + +```sql +extractAll(haystack, pattern) +``` + +## extractAllGroupsHorizontal + +使用`pattern`正则表达式匹配`haystack`字符串的所有组。 + +返回一个元素为数组的数组，其中第一个数组包含与第一组匹配的所有片段，第二个数组包含与第二组匹配的所有片段，依此类推。 + +这个函数相比 [extractAllGroupsVertical](#extractallgroups-vertical)更慢。 + +**语法** + +``` sql +extractAllGroupsHorizontal(haystack, pattern) +``` + +**参数** + +- `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含 group，每个 group 用括号括起来。 如果 `pattern` 不包含 group 则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). + +**返回值** + +- 数据类型: [Array](../../sql-reference/data-types/array.md). + +如果`haystack`不匹配`pattern`正则表达式，则返回一个空数组的数组。 + +**示例** + +``` sql +SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); +``` + +结果： + +``` text +┌─extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)')─┐ +│ [['abc','def','ghi'],['111','222','333']] │ +└──────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## extractAllGroupsVertical + +使用正则表达式 `pattern`匹配字符串`haystack`中的所有group。返回一个数组，其中每个数组包含每个group的匹配片段。片段按照在`haystack`中出现的顺序进行分组。 + +**语法** + +``` sql +extractAllGroupsVertical(haystack, pattern) +``` + +**参数** + +- `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含group，每个group用括号括起来。 如果 `pattern` 不包含group则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). + +**返回值** + +- 数据类型: [Array](../../sql-reference/data-types/array.md). + +如果`haystack`不匹配`pattern`正则表达式，则返回一个空数组。 + +**示例** + +``` sql +SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); +``` + +结果： + +``` text +┌─extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)')─┐ +│ [['abc','111'],['def','222'],['ghi','333']] │ +└────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## like {#like} + +返回字符串 `haystack` 是否匹配 LIKE 表达式 `pattern`。 + +一个 LIKE 表达式可以包含普通字符和以下元字符： + +- `%` 表示任意数量的任意字符（包括零个字符）。 +- `_` 表示单个任意字符。 +- `\` 用于转义文字 `%`, `_` 和 `\`。 + +匹配基于 UTF-8，例如 `_` 匹配 Unicode 代码点 `¥`，它使用两个字节以 UTF-8 表示。 + +如果 `haystack` 或 `LIKE` 表达式不是有效的 UTF-8，则行为是未定义的。 + +不会自动执行 Unicode 规范化，您可以使用[normalizeUTF8*()](https://clickhouse.com/docs/zh/sql-reference/functions/string-functions/) 函数来执行此操作。 + +如果需要匹配字符 `%`, `_` 和 `/`（这些是 LIKE 元字符），请在其前面加上反斜杠：`\%`, `\_` 和 `\\`。 +如果在非 `%`, `_` 或 `\` 字符前使用反斜杠，则反斜杠将失去其特殊含义（即被解释为字面值）。 +请注意，ClickHouse 要求字符串中使用反斜杠 [也需要被转义](../syntax.md#string), 因此您实际上需要编写 `\\%`、`\\_` 和 `\\\\`。 + + +对于形式为 `%needle%` 的 LIKE 表达式，函数的性能与 `position` 函数相同。 +所有其他 LIKE 表达式都会被内部转换为正则表达式，并以与函数 `match` 相似的性能执行。 + +**语法** + +```sql +like(haystack, pattern) +``` + +别名： `haystack LIKE pattern` (operator) + +## notLike {#notlike} + +类似于 `like` 但是返回相反的结果。 + +别名： `haystack NOT LIKE pattern` (operator) + +## ilike + +类似于 `like` 但是不区分大小写。 + +别名： `haystack ILIKE pattern` (operator) + +## notILike + +类似于 `ilike` 但是返回相反的结果。 + +别名： `haystack NOT ILIKE pattern` (operator) + +## ngramDistance + +计算字符串 `haystack` 和子字符串 `needle` 的 4-gram 距离。 为此，它计算两个 4-gram 多重集之间的对称差异，并通过它们的基数之和对其进行标准化。返回 0 到 1 之间的 Float32 浮点数。返回值越小，代表字符串越相似. 如果参数 `needle` or `haystack` 是常数且大小超过 32Kb，则抛出异常。如果参数 `haystack` 或 `needle` 是非常数且大小超过 32Kb ，则返回值恒为 1。 + +函数 `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + +**语法** + +```sql +ngramDistance(haystack, needle) +``` + +## ngramSearch + +类似于`ngramDistance`，但计算`needle`字符串和 `haystack` 字符串之间的非对称差异，即来自 `needle` 的 n-gram 数量减去由`needle`数量归一化的 n-gram 的公共数量 n-gram。返回 0 到 1 之间的 Float32 浮点数。结果越大，`needle` 越有可能在 `haystack` 中。该函数对于模糊字符串搜索很有用。另请参阅函数 `soundex``。 + +函数 `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + :::note -如要关闭所有hyperscan函数的使用，请设置`SET allow_hyperscan = 0;`。 +UTF-8 变体使用了 3-gram 距离。这些并不是完全公平的 n-gram 距离。我们使用 2 字节的哈希函数来哈希 n-gram，然后计算这些哈希表之间的(非)对称差异——可能会发生冲突。在使用 UTF-8 大小写不敏感格式时，我们并不使用公平的 `tolower` 函数——我们将每个码点字节的第 5 位（从零开始）和第零字节的第一个比特位位置为零（如果该串的大小超过一个字节）——这对拉丁字母和大部分西里尔字母都有效。 ::: -## 提取（大海捞针，图案) {#extracthaystack-pattern} +**语法** -使用正则表达式截取字符串。如果’haystack’与’pattern’不匹配，则返回空字符串。如果正则表达式中不包含子模式，它将获取与整个正则表达式匹配的子串。否则，它将获取与第一个子模式匹配的子串。 +```sql +ngramSearch(haystack, needle) +``` -## extractAll（大海捞针，图案) {#extractallhaystack-pattern} +## countSubstrings -使用正则表达式提取字符串的所有片段。如果’haystack’与’pattern’正则表达式不匹配，则返回一个空字符串。否则返回所有与正则表达式匹配的字符串数组。通常，行为与’extract’函数相同（它采用第一个子模式，如果没有子模式，则采用整个表达式）。 +返回字符串 `haystack` 中子字符串 `needle` 出现的次数。 -## 像（干草堆，模式），干草堆像模式运算符 {#likehaystack-pattern-haystack-like-pattern-operator} +函数 `countSubstringsCaseInsensitive` 和 `countSubstringsCaseInsensitiveUTF8` 提供此函数的不区分大小写以及 UTF-8 变体。 -检查字符串是否与简单正则表达式匹配。 -正则表达式可以包含的元符号有`％`和`_`。 +**语法** -`%` 表示任何字节数（包括零字符）。 +``` sql +countSubstrings(haystack, needle[, start_pos]) +``` -`_` 表示任何一个字节。 +**参数** -可以使用反斜杠（`\`）来对元符号进行转义。请参阅«match»函数说明中有关转义的说明。 +- `haystack` — 被搜索的字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 用于搜索的模式子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` – 在字符串`haystack` 中开始检索的位置(从 1 开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选。 -对于像`％needle％`这样的正则表达式，改函数与`position`函数一样快。 -对于其他正则表达式，函数与’match’函数相同。 +**返回值** -## 不喜欢（干草堆，模式），干草堆不喜欢模式运算符 {#notlikehaystack-pattern-haystack-not-like-pattern-operator} +- 子字符串出现的次数。 -与’like’函数返回相反的结果。 +数据类型: [UInt64](../../sql-reference/data-types/int-uint.md). -## 大海捞针) {#ngramdistancehaystack-needle} +**示例** -基于4-gram计算`haystack`和`needle`之间的距离：计算两个4-gram集合之间的对称差异，并用它们的基数和对其进行归一化。返回0到1之间的任何浮点数 – 越接近0则表示越多的字符串彼此相似。如果常量的`needle`或`haystack`超过32KB，函数将抛出异常。如果非常量的`haystack`或`needle`字符串超过32Kb，则距离始终为1。 +``` sql +SELECT countSubstrings('aaaa', 'aa'); +``` -对于不区分大小写的搜索或/和UTF-8格式，使用函数`ngramDistanceCaseInsensitive，ngramDistanceUTF8，ngramDistanceCaseInsensitiveUTF8`。 +结果： -## ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人) {#ngramsearchhaystack-needle} +``` text +┌─countSubstrings('aaaa', 'aa')─┐ +│ 2 │ +└───────────────────────────────┘ +``` -与`ngramDistance`相同，但计算`needle`和`haystack`之间的非对称差异——`needle`的n-gram减去`needle`归一化n-gram。可用于模糊字符串搜索。 +示例，使用参数 `start_pos` : -对于不区分大小写的搜索或/和UTF-8格式，使用函数`ngramSearchCaseInsensitive，ngramSearchUTF8，ngramSearchCaseInsensitiveUTF8`。 +```sql +SELECT countSubstrings('abc___abc', 'abc', 4); +``` -:::note -对于UTF-8，我们使用3-gram。所有这些都不是完全公平的n-gram距离。我们使用2字节哈希来散列n-gram，然后计算这些哈希表之间的（非）对称差异 - 可能会发生冲突。对于UTF-8不区分大小写的格式，我们不使用公平的`tolower`函数 - 我们将每个Unicode字符字节的第5位（从零开始）和字节的第一位归零 - 这适用于拉丁语，主要用于所有西里尔字母。 -::: +结果： + +``` text +┌─countSubstrings('abc___abc', 'abc', 4)─┐ +│ 1 │ +└────────────────────────────────────────┘ +``` + +## countMatches + +返回正则表达式 `pattern` 在 `haystack` 中成功匹配的次数。 + +**语法** + +``` sql +countMatches(haystack, pattern) +``` + +**参数** + +- `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax)） 数据类型为[String](../../sql-reference/data-types/string.md). + +**返回值** + +- 匹配次数。 + +数据类型: [UInt64](../../sql-reference/data-types/int-uint.md). + +**示例** + +``` sql +SELECT countMatches('foobar.com', 'o+'); +``` + +结果： + +``` text +┌─countMatches('foobar.com', 'o+')─┐ +│ 2 │ +└──────────────────────────────────┘ +``` + +``` sql +SELECT countMatches('aaaa', 'aa'); +``` + +结果： + +``` text +┌─countMatches('aaaa', 'aa')────┐ +│ 2 │ +└───────────────────────────────┘ +``` + +## countMatchesCaseInsensitive + +类似于 `countMatches(haystack, pattern)` 但是不区分大小写。 + +## regexpExtract + +提取匹配正则表达式模式的字符串`haystack`中的第一个字符串，并对应于正则表达式组索引。 + +**语法** + +``` sql +regexpExtract(haystack, pattern[, index]) +``` + +别名： `REGEXP_EXTRACT(haystack, pattern[, index])`. + +**参数** + +- `haystack` — 被匹配字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `pattern` — 正则表达式，必须是常量。类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `index` – 一个大于等于 0 的整数，默认为 1 ，它代表要提取哪个正则表达式组。 [UInt or Int](../../sql-reference/data-types/int-uint.md) 可选。 + +**返回值** + +`pattern`可以包含多个正则组, `index` 代表要提取哪个正则表达式组。如果 `index` 为 0，则返回整个匹配的字符串。 + +数据类型: `String`. + +**示例** + +``` sql +SELECT + regexpExtract('100-200', '(\\d+)-(\\d+)', 1), + regexpExtract('100-200', '(\\d+)-(\\d+)', 2), + regexpExtract('100-200', '(\\d+)-(\\d+)', 0), + regexpExtract('100-200', '(\\d+)-(\\d+)'); +``` + +结果： + +``` text +┌─regexpExtract('100-200', '(\\d+)-(\\d+)', 1)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)', 2)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)', 0)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)')─┐ +│ 100 │ 200 │ 100-200 │ 100 │ +└──────────────────────────────────────────────┴──────────────────────────────────────────────┴──────────────────────────────────────────────┴───────────────────────────────────────────┘ +``` + +## hasSubsequence + +如果`needle`是`haystack`的子序列，返回1，否则返回0。 +子序列是从给定字符串中删除零个或多个元素而不改变剩余元素的顺序得到的序列。 + +**语法** + +``` sql +hasSubsequence(haystack, needle) +``` + +**参数** + +- `haystack` — 被搜索的字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 搜索子序列，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). + +**返回值** + +- 1, 如果`needle`是`haystack`的子序列 +- 0, 如果`needle`不是`haystack`的子序列 + +数据类型: `UInt8`. + +**示例** + +``` sql +SELECT hasSubsequence('garbage', 'arg') ; +``` + +结果： + +``` text +┌─hasSubsequence('garbage', 'arg')─┐ +│ 1 │ +└──────────────────────────────────┘ +``` + +## hasSubsequenceCaseInsensitive +类似于 [hasSubsequence](#hasSubsequence) 但是不区分大小写。 + +## hasSubsequenceUTF8 + +类似于 [hasSubsequence](#hasSubsequence) 但是假定 `haystack` 和 `needle` 是 UTF-8 编码的字符串。 + +## hasSubsequenceCaseInsensitiveUTF8 + +类似于 [hasSubsequenceUTF8](#hasSubsequenceUTF8) 但是不区分大小写。 diff --git a/packages/clickhouse-common-static-dbg.yaml b/packages/clickhouse-common-static-dbg.yaml index 96de4c17d88..74b7fa8381b 100644 --- a/packages/clickhouse-common-static-dbg.yaml +++ b/packages/clickhouse-common-static-dbg.yaml @@ -30,10 +30,6 @@ conflicts: contents: - src: root/usr/lib/debug/usr/bin/clickhouse.debug dst: /usr/lib/debug/usr/bin/clickhouse.debug -- src: root/usr/lib/debug/usr/bin/clickhouse-odbc-bridge.debug - dst: /usr/lib/debug/usr/bin/clickhouse-odbc-bridge.debug -- src: root/usr/lib/debug/usr/bin/clickhouse-library-bridge.debug - dst: /usr/lib/debug/usr/bin/clickhouse-library-bridge.debug # docs - src: ../AUTHORS dst: /usr/share/doc/clickhouse-common-static-dbg/AUTHORS diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml index 238126f95fd..db330f808e1 100644 --- a/packages/clickhouse-common-static.yaml +++ b/packages/clickhouse-common-static.yaml @@ -34,14 +34,8 @@ suggests: contents: - src: root/usr/bin/clickhouse dst: /usr/bin/clickhouse -- src: root/usr/bin/clickhouse-diagnostics - dst: /usr/bin/clickhouse-diagnostics - src: root/usr/bin/clickhouse-extract-from-config dst: /usr/bin/clickhouse-extract-from-config -- src: root/usr/bin/clickhouse-library-bridge - dst: /usr/bin/clickhouse-library-bridge -- src: root/usr/bin/clickhouse-odbc-bridge - dst: /usr/bin/clickhouse-odbc-bridge - src: root/usr/share/bash-completion/completions dst: /usr/share/bash-completion/completions - src: root/usr/share/clickhouse diff --git a/packages/clickhouse-library-bridge.yaml b/packages/clickhouse-library-bridge.yaml new file mode 100644 index 00000000000..d041e7a26db --- /dev/null +++ b/packages/clickhouse-library-bridge.yaml @@ -0,0 +1,35 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-library-bridge" +description: | + ClickHouse Library Bridge - is a separate process for loading libraries for the 'library' dictionary sources and the CatBoost library. + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + +# Common packages config +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" +maintainer: "ClickHouse Dev Team " +deb: + fields: + Source: clickhouse + +# Package specific content +contents: +- src: root/usr/bin/clickhouse-library-bridge + dst: /usr/bin/clickhouse-library-bridge +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-library-bridge/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-library-bridge/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-library-bridge/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-library-bridge/README.md diff --git a/packages/clickhouse-odbc-bridge.yaml b/packages/clickhouse-odbc-bridge.yaml new file mode 100644 index 00000000000..98c459c8c26 --- /dev/null +++ b/packages/clickhouse-odbc-bridge.yaml @@ -0,0 +1,35 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-odbc-bridge" +description: | + ClickHouse ODBC Bridge - is a separate process for loading ODBC drivers and interacting with external databases using the ODBC protocol. + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + +# Common packages config +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" +maintainer: "ClickHouse Dev Team " +deb: + fields: + Source: clickhouse + +# Package specific content +contents: +- src: root/usr/bin/clickhouse-odbc-bridge + dst: /usr/bin/clickhouse-odbc-bridge +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-odbc-bridge/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-odbc-bridge/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-odbc-bridge/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-odbc-bridge/README.md diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index d945fdf4a6f..0d91de2dad8 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -268,10 +268,6 @@ if (ENABLE_TESTS) add_dependencies(clickhouse-bundle clickhouse-tests) endif() -if (ENABLE_FUZZING) - add_compile_definitions(FUZZING_MODE=1) -endif () - if (TARGET ch_contrib::protobuf) get_property(google_proto_files TARGET ch_contrib::protobuf PROPERTY google_proto_files) foreach (proto_file IN LISTS google_proto_files) diff --git a/programs/bash-completion/completions/clickhouse b/programs/bash-completion/completions/clickhouse index ff0a60c60be..3c895a66075 100644 --- a/programs/bash-completion/completions/clickhouse +++ b/programs/bash-completion/completions/clickhouse @@ -3,7 +3,7 @@ function _clickhouse_get_utils() { local cmd=$1 && shift - "$cmd" --help |& awk '/^clickhouse.*args/ { print $2 }' + "$cmd" help |& awk '/^clickhouse.*args/ { print $2 }' } function _complete_for_clickhouse_entrypoint_bin() diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 45dadfef774..eecc352d073 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -34,6 +34,7 @@ #include #include #include +#include /** A tool for evaluating ClickHouse performance. @@ -623,7 +624,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ; Settings settings; - settings.addProgramOptions(desc); + addProgramOptions(settings, desc); boost::program_options::variables_map options; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 4203e4738dd..053b29481ce 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -7,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -19,12 +17,13 @@ #include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include #include #include @@ -45,8 +44,6 @@ #include -#include - #include #include #include @@ -933,12 +930,12 @@ void Client::addOptions(OptionsDescription & options_description) ("config,c", po::value(), "config-file path (another shorthand)") ("connection", po::value(), "connection to use (from the client config), by default connection name is hostname") ("secure,s", "Use TLS connection") - ("no-secure,s", "Don't use TLS connection") + ("no-secure", "Don't use TLS connection") ("user,u", po::value()->default_value("default"), "user") ("password", po::value(), "password") ("ask-password", "ask-password") - ("ssh-key-file", po::value(), "File containing ssh private key needed for authentication. If not set does password authentication.") - ("ssh-key-passphrase", po::value(), "Passphrase for imported ssh key.") + ("ssh-key-file", po::value(), "File containing the SSH private key for authenticate with the server.") + ("ssh-key-passphrase", po::value(), "Passphrase for the SSH private key specified by --ssh-key-file.") ("quota_key", po::value(), "A string to differentiate quotas when the user have keyed quotas configured on server") ("max_client_network_bandwidth", po::value(), "the maximum speed of data exchange over the network for the client in bytes per second.") @@ -953,6 +950,7 @@ void Client::addOptions(OptionsDescription & options_description) ("opentelemetry-tracestate", po::value(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation") ("no-warnings", "disable warnings when client connects to server") + /// TODO: Left for compatibility as it's used in upgrade check, remove after next release and use server setting ignore_drop_queries_probability ("fake-drop", "Ignore all DROP queries, should be used only for testing") ("accept-invalid-certificate", "Ignore certificate verification errors, equal to config parameters openSSL.client.invalidCertificateHandler.name=AcceptCertificateHandler and openSSL.client.verificationMode=none") ; @@ -1096,7 +1094,7 @@ void Client::processOptions(const OptionsDescription & options_description, if (options.count("no-warnings")) config().setBool("no-warnings", true); if (options.count("fake-drop")) - fake_drop = true; + config().setString("ignore_drop_queries_probability", "1"); if (options.count("accept-invalid-certificate")) { config().setString("openSSL.client.invalidCertificateHandler.name", "AcceptCertificateHandler"); @@ -1138,13 +1136,6 @@ void Client::processOptions(const OptionsDescription & options_description, } -static bool checkIfStdoutIsRegularFile() -{ - struct stat file_stat; - return fstat(STDOUT_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode); -} - - void Client::processConfig() { if (!queries.empty() && config().has("queries-file")) @@ -1180,34 +1171,7 @@ void Client::processConfig() pager = config().getString("pager", ""); - is_default_format = !config().has("vertical") && !config().has("format"); - if (is_default_format && checkIfStdoutIsRegularFile()) - { - is_default_format = false; - std::optional format_from_file_name; - format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDOUT_FILENO); - format = format_from_file_name ? *format_from_file_name : "TabSeparated"; - } - else if (config().has("vertical")) - format = config().getString("format", "Vertical"); - else - format = config().getString("format", is_interactive ? "PrettyCompact" : "TabSeparated"); - - format_max_block_size = config().getUInt64("format_max_block_size", - global_context->getSettingsRef().max_block_size); - - insert_format = "Values"; - - /// Setting value from cmd arg overrides one from config - if (global_context->getSettingsRef().max_insert_block_size.changed) - { - insert_format_max_block_size = global_context->getSettingsRef().max_insert_block_size; - } - else - { - insert_format_max_block_size = config().getUInt64("insert_format_max_block_size", - global_context->getSettingsRef().max_insert_block_size); - } + setDefaultFormatsFromConfiguration(); global_context->setClientName(std::string(DEFAULT_CLIENT_NAME)); global_context->setQueryKindInitial(); diff --git a/programs/diagnostics/.gitignore b/programs/diagnostics/.gitignore deleted file mode 100644 index 5e0b0165f38..00000000000 --- a/programs/diagnostics/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -# If you prefer the allow list template instead of the deny list, see community template: -# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file -go.work - -.idea -clickhouse-diagnostics -output -vendor -bin -profile.cov -clickhouse-diagnostics.yml -dist/ diff --git a/programs/diagnostics/CONTRIBUTION.md b/programs/diagnostics/CONTRIBUTION.md deleted file mode 100644 index 00fb073cefe..00000000000 --- a/programs/diagnostics/CONTRIBUTION.md +++ /dev/null @@ -1,49 +0,0 @@ -# Contribution - -We keep things simple. Execute all commands in this folder. - -## Requirements - -- docker - tested on version 20.10.12. -- golang >= go1.17.6 - -## Building - -Creates a binary `clickhouse-diagnostics` in the local folder. Build will be versioned according to a timestamp. For a versioned release see [Releasing](#releasing). - -```bash -make build -``` - -## Linting - -We use [golangci-lint](https://golangci-lint.run/). We use a container to run so no need to install. - -```bash -make lint-go -``` - -## Running Tests - -```bash -make test -``` - -For a coverage report, - -```bash -make test-coverage -``` - -## Adding Collectors - -TODO - - -## Adding Outputs - -TODO - -## Frames - -## Parameter Types diff --git a/programs/diagnostics/Makefile b/programs/diagnostics/Makefile deleted file mode 100644 index 2e85002b871..00000000000 --- a/programs/diagnostics/Makefile +++ /dev/null @@ -1,65 +0,0 @@ -GOCMD=go -GOTEST=$(GOCMD) test -BINARY_NAME=clickhouse-diagnostics -BUILD_DIR=dist - -TIMESTAMP := $(shell date +%Y%m%d-%H%M) -COMMIT := $(shell git rev-parse --short HEAD) -MODULE := github.com/ClickHouse/ClickHouse/programs/diagnostics -VERSION := v.dev-${TIMESTAMP} -DEVLDFLAGS = -ldflags "-X ${MODULE}/cmd.Version=${VERSION} -X ${MODULE}/cmd.Commit=${COMMIT}" - -# override with env variable to test other versions e.g. 21.11.10.1 -CLICKHOUSE_VERSION ?= latest - -GREEN := $(shell tput -Txterm setaf 2) -YELLOW := $(shell tput -Txterm setaf 3) -WHITE := $(shell tput -Txterm setaf 7) -CYAN := $(shell tput -Txterm setaf 6) -RESET := $(shell tput -Txterm sgr0) - -.PHONY: all test build vendor release lint-go test-coverages dep - -all: help - -release: ## Release is delegated to goreleaser - $(shell goreleaser release --rm-dist) - -## Build: -build: ## Build a binary for local use - # timestamped version - $(GOCMD) build ${DEVLDFLAGS} -o $(BINARY_NAME) ./cmd/clickhouse-diagnostics - -clean: ## Remove build related file - rm ${BINARY_NAME} - rm -f checkstyle-report.xml ./coverage.xml ./profile.cov - -vendor: ## Copy of all packages needed to support builds and tests in the vendor directory - $(GOCMD) mod vendor - -test: ## Run the tests of the project - CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -v -race `go list ./... | grep -v ./internal/platform/test` - -test-no-docker: ## Don't run tests depending on dockerd - CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -v -race -tags no_docker `go list ./... | grep -v ./internal/platform/test` - -lint-go: ## Use golintci-lint - docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:latest-alpine golangci-lint run - -test-coverage: ## Run the tests of the project and export the coverage - CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -cover -covermode=count -coverprofile=profile.cov `go list ./... | grep -v ./internal/platform/test` - $(GOCMD) tool cover -func profile.cov - -dep: - $(shell go mod download) - -help: ## Show this help. - @echo '' - @echo 'Usage:' - @echo ' ${YELLOW}make${RESET} ${GREEN}${RESET}' - @echo '' - @echo 'Targets:' - @awk 'BEGIN {FS = ":.*?## "} { \ - if (/^[a-zA-Z_-]+:.*?##.*$$/) {printf " ${YELLOW}%-20s${GREEN}%s${RESET}\n", $$1, $$2} \ - else if (/^## .*$$/) {printf " ${CYAN}%s${RESET}\n", substr($$1,4)} \ - }' $(MAKEFILE_LIST) diff --git a/programs/diagnostics/README.md b/programs/diagnostics/README.md deleted file mode 100644 index f800bb0648e..00000000000 --- a/programs/diagnostics/README.md +++ /dev/null @@ -1,167 +0,0 @@ -# Clickhouse Diagnostics Tool - -## Purpose - -This tool provides a means of obtaining a diagnostic bundle from a ClickHouse instance. This bundle can be provided to your nearest ClickHouse support provider in order to assist with the diagnosis of issues. - -## Design Philosophy - -- **No local dependencies** to run. We compile to a platform-independent binary, hence Go. -- **Minimize resource overhead**. Improvements always welcome. -- **Extendable framework**. At its core, the tool provides collectors and outputs. Collectors are independent and are responsible for collecting a specific dataset e.g. system configuration. Outputs produce the diagnostic bundle in a specific format. It should be trivial to add both for contributors. See [Collectors](#collectors) and [Outputs](#outputs) for more details. -- **Convertible output formats**. Outputs produce diagnostic bundles in different formats e.g. archive, simple report etc. Where possible, it should be possible to convert between these formats. For example, an administrator may provide a bundle as an archive to their support provider who in turn wishes to visualise this as a report or even in ClickHouse itself... -- **Something is better than nothing**. Collectors execute independently. We never fail a collection because one fails - preferring to warn the user only. There are good reasons for a collector failure e.g. insufficient permissions or missing data. -- **Execute anywhere** - Ideally, this tool is executed on a ClickHouse host. Some collectors e.g. configuration file collection or system information, rely on this. However, collectors will obtain as much information remotely from the database as possible if executed remotely from the cluster - warning where collection fails. **We do currently require ClickHouse to be running, connecting over the native port**. - -We recommend reading [Permissions, Warnings & Locality](#permissions-warnings--locality). - -## Usage - -### Collection - -The `collect` command allows the collection of a diagnostic bundle. In its simplest form, assuming ClickHouse is running locally on default ports with no password: - -```bash -clickhouse-diagnostics collect -``` - -This will use the default collectors and the simple output. This output produces a timestamped archive bundle in `gz` format in a sub folder named after the host. This folder name can be controlled via the parameter `--id` or configured directly for the simple output parameter `output.simple.folder` (this allows a specific directory to be specified). - -Collectors, Outputs and ClickHouse connection credentials can be specified as shown below: - -```bash -clickhouse-diagnostics collect --password random --username default --collector=system_db,system --output=simple --id my_cluster_name -``` - -This collects the system database and host information from the cluster running locally. The archive bundle will be produced under a folder `my_cluster_name`. - -For further details, use the in built help (the commands below are equivalent): - -```bash -clickhouse-diagnostics collect --help -./clickhouse-diagnostics help collect -``` - -### Help & Finding parameters for collectors & outputs - -Collectors and outputs have their own parameters not listed under the help for the command for the `collect` command. These can be identified using the `help` command. Specifically, - -For more information about a specific collector. - -```bash -Use "clickhouse-diagnostics help --collector [collector]" -``` - -For more information about a specific output. - -```bash -Use "clickhouse-diagnostics help --output [output]" -``` - -### Convert - -Coming soon to a cluster near you... - -## Collectors - -We currently support the following collectors. A `*` indicates this collector is enabled by default: - -- `system_db*` - Collects all tables in the system database, except those which have been excluded and up to a specified row limit. -- `system*` - Collects summary OS and hardware statistics for the host. -- `config*` - Collects the ClickHouse configuration from the local filesystem. A best effort is made using process information if ClickHouse is not installed locally. `include_path` are also considered. -- `db_logs*` - Collects the ClickHouse logs directly from the database. -- `logs*` - Collects the ClickHouse logs directly from the database. -- `summary*` - Collects summary statistics on the database based on a set of known useful queries. This represents the easiest collector to extend - contributions are welcome to this set which can be found [here](https://github.com/ClickHouse/ClickHouse/blob/master/programs/diagnostics/internal/collectors/clickhouse/queries.json). -- `file` - Collects files based on glob patterns. Does not collect directories. To preview files which will be collected try, `clickhouse-diagnostics collect --collectors=file --collector.file.file_pattern= --output report` -- `command` - Collects the output of a user specified command. To preview output, `clickhouse-diagnostics collect --collectors=command --collector.command.command="" --output report` -- `zookeeper_db` - Collects information about zookeeper using the `system.zookeeper` table, recursively iterating the zookeeper tree/table. Note: changing the default parameter values can cause extremely high load to be placed on the database. Use with caution. By default, uses the glob `/clickhouse/{task_queue}/**` to match zookeeper paths and iterates to a max depth of 8. - -## Outputs - -We currently support the following outputs. The `simple` output is currently the default: - -- `simple` - Writes out the diagnostic bundle as files in a structured directory, optionally producing a compressed archive. -- `report` - Writes out the diagnostic bundle to the terminal as a simple report. Supports an ascii table format or markdown. -- `clickhouse` - **Under development**. This will allow a bundle to be stored in a cluster allowing visualization in common tooling e.g. Grafana. - -## Simple Output - -Since the `simple` output is the default we provide additional details here. -This output produces a timestamped archive by default in `gz` format under a directory created with either the hostname of the specified collection `--id`. As shown below, a specific folder can also be specified. Compression can also be disabled, leaving just the contents of the folder: - -```bash -./clickhouse-diagnostics help --output simple - -Writes out the diagnostic bundle as files in a structured directory, optionally producing a compressed archive. - -Usage: - --output=simple [flags] - -Flags: - --output.simple.directory string Directory in which to create dump. Defaults to the current directory. (default "./") - --output.simple.format string Format of exported files (default "csv") - --output.simple.skip_archive Don't compress output to an archive -``` - -The archive itself contains a folder for each collector. Each collector can potentially produce many discrete sets of data, known as frames. Each of these typically results in a single file within the collector's folder. For example, each query for the `summary` collector results in a correspondingly named file within the `summary` folder. - -## Permissions, Warnings & Locality - -Some collectors either require specific permissions for complete collection or should be executed on a ClickHouse host. We aim to collate these requirements below: - -- `system_db` - This collect aims to collect all tables in the `system` database. Some tables may fail if certain features are not enabled. Specifically,[allow_introspection_functions](https://clickhouse.com/docs/en/operations/settings/settings/#settings-allow_introspection_functions) is required to collect the `stack_traces` table. [access_management](https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting) must be set for the ClickHouse user specified for collection, to permit access to access management tables e.g. `quota_usage`. -- `db_logs`- The ClickHouse user must have access to the tables `query_log`,`query_thread_log` and `text_log`. -- `logs` - The system user under which the tool is executed must have access to the logs directory. It must therefore also be executed on the target ClickHouse server directly for this collector work. In cases where the logs directory is not a default location e.g. `/var/log/clickhouse-server` we will attempt to establish the location from the ClickHouse configuration. This requires permissions to read the configuration files - which in most cases requires specific permissions to be granted to the run user if you are not comfortable executing the tool under sudo or the `clickhouse` user. -- `summary`- This collector executes pre-recorded queries. Some of these read tables concerning access management, thus requiring the ClickHouse user to have the [access_management](https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting) permission. -- `config` - This collector reads and copies the local configuration files. It thus requires permissions to read the configuration files - which in most cases requires specific permissions to be granted to the run user if you are not comfortable executing the tool under sudo or the `clickhouse` user. - -**If a collector cannot collect specific data because of either execution location or permissions, it will log a warning to the terminal.** - -## Logging - -All logs are output to `stderr`. `stdout` is used exclusively for outputs to print information. - -## Configuration file - -In addition to supporting parameters via the command line, a configuration file can be specified via the `--config`, `-f` flag. - -By default, we look for a configuration file `clickhouse-diagnostics.yml` in the same directory as the binary. If not present, we revert to command line flags. - -**Values set via the command line values always take precedence over those in the configuration file.** - -All parameters can be set via the configuration file and can in most cases be converted to a yaml hierarchy, where periods indicate a nesting. For example, - -`--collector.system_db.row_limit=1` - -becomes - -```yaml -collector: - system_db: - row_limit: 1 -``` - -The following exceptions exist to avoid collisions: - -| Command | Parameter | Configuration File | -|---------|------------|--------------------| -| collect | output | collect.output | -| collect | collectors | collect.collectors | - -## FAQ - -1. Does the collector need root permissions? - - No. However, to read some local files e.g. configurations, the tool should be executed as the `clickhouse` user. - -2. What ClickHouse database permissions does the collector need? - - Read permissions on all system tables are required in most cases - although only specific collectors need this. [Access management permissions]((https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting)) will ensure full collection. - -3. Is any processing done on logs for anonimization purposes? - - Currently no. ClickHouse should not log sensitive information to logs e.g. passwords. - -4. Is sensitive information removed from configuration files e.g. passwords? - - Yes. We remove both passwords and hashed passwords. Please raise an issue if you require further information to be anonimized. We appreciate this is a sensitive topic. diff --git a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go deleted file mode 100644 index 0a849a9f520..00000000000 --- a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd" -) - -func main() { - cmd.Execute() -} diff --git a/programs/diagnostics/cmd/collect.go b/programs/diagnostics/cmd/collect.go deleted file mode 100644 index 503d8e41fb7..00000000000 --- a/programs/diagnostics/cmd/collect.go +++ /dev/null @@ -1,159 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "github.com/spf13/viper" -) - -var id string -var output = params.StringOptionsVar{ - Options: outputs.GetOutputNames(), - Value: "simple", -} - -// access credentials -var host string -var port uint16 -var username string -var password string - -var collectorNames = params.StringSliceOptionsVar{ - Options: collectors.GetCollectorNames(false), - Values: collectors.GetCollectorNames(true), -} - -// holds the collector params passed by the cli -var collectorParams params.ParamMap - -// holds the output params passed by the cli -var outputParams params.ParamMap - -const collectHelpTemplate = `Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} - {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} - -Aliases: - {{.NameAndAliases}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}}{{if .HasAvailableSubCommands}} - -Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} - -Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} - -Global Flags: -{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}} - -Additional help topics: - Use "{{.CommandPath}} [command] --help" for more information about a command. - Use "{{.Parent.Name}} help --collector [collector]" for more information about a specific collector. - Use "{{.Parent.Name}} help --output [output]" for more information about a specific output. -` - -func init() { - collectCmd.Flags().StringVar(&id, "id", getHostName(), "Id of diagnostic bundle") - - // access credentials - collectCmd.Flags().StringVar(&host, "host", "localhost", "ClickHouse host") - collectCmd.Flags().Uint16VarP(&port, "port", "p", 9000, "ClickHouse native port") - collectCmd.Flags().StringVarP(&username, "username", "u", "", "ClickHouse username") - collectCmd.Flags().StringVar(&password, "password", "", "ClickHouse password") - // collectors and outputs - collectCmd.Flags().VarP(&output, "output", "o", fmt.Sprintf("Output Format for the diagnostic Bundle, options: [%s]\n", strings.Join(output.Options, ","))) - collectCmd.Flags().VarP(&collectorNames, "collectors", "c", fmt.Sprintf("Collectors to use, options: [%s]\n", strings.Join(collectorNames.Options, ","))) - - collectorConfigs, err := collectors.BuildConfigurationOptions() - if err != nil { - log.Fatal().Err(err).Msg("Unable to build collector configurations") - } - collectorParams = params.NewParamMap(collectorConfigs) - - outputConfigs, err := outputs.BuildConfigurationOptions() - if err != nil { - log.Fatal().Err(err).Msg("Unable to build output configurations") - } - params.AddParamMapToCmd(collectorParams, collectCmd, "collector", true) - - outputParams = params.NewParamMap(outputConfigs) - params.AddParamMapToCmd(outputParams, collectCmd, "output", true) - - collectCmd.SetFlagErrorFunc(handleFlagErrors) - collectCmd.SetHelpTemplate(collectHelpTemplate) - rootCmd.AddCommand(collectCmd) -} - -var collectCmd = &cobra.Command{ - Use: "collect", - Short: "Collect a diagnostic bundle", - Long: `Collect a ClickHouse diagnostic bundle for a specified ClickHouse instance`, - PreRun: func(cmd *cobra.Command, args []string) { - bindFlagsToConfig(cmd) - }, - Example: fmt.Sprintf(`%s collect --username default --collector=%s --output=simple`, rootCmd.Name(), strings.Join(collectorNames.Options[:2], ",")), - Run: func(cmd *cobra.Command, args []string) { - log.Info().Msgf("executing collect command with %v collectors and %s output", collectorNames.Values, output.Value) - outputConfig := params.ConvertParamsToConfig(outputParams)[output.Value] - runConfig := internal.NewRunConfiguration(id, host, port, username, password, output.Value, outputConfig, collectorNames.Values, params.ConvertParamsToConfig(collectorParams)) - internal.Capture(runConfig) - os.Exit(0) - }, -} - -func getHostName() string { - name, err := os.Hostname() - if err != nil { - name = "clickhouse-diagnostics" - } - return name -} - -// these flags are nested under the cmd name in the config file to prevent collisions -var flagsToNest = []string{"output", "collectors"} - -// this saves us binding each command manually to viper -func bindFlagsToConfig(cmd *cobra.Command) { - cmd.Flags().VisitAll(func(f *pflag.Flag) { - err := viper.BindEnv(f.Name, fmt.Sprintf("%s_%s", envPrefix, - strings.ToUpper(strings.Replace(f.Name, ".", "_", -1)))) - if err != nil { - log.Error().Msgf("Unable to bind %s to config", f.Name) - } - configFlagName := f.Name - if utils.Contains(flagsToNest, f.Name) { - configFlagName = fmt.Sprintf("%s.%s", cmd.Use, configFlagName) - } - err = viper.BindPFlag(configFlagName, f) - if err != nil { - log.Error().Msgf("Unable to bind %s to config", f.Name) - } - // here we prefer the config value when the param is not set on the cmd line - if !f.Changed && viper.IsSet(configFlagName) { - val := viper.Get(configFlagName) - log.Debug().Msgf("Setting parameter %s from configuration file", f.Name) - err = cmd.Flags().Set(f.Name, fmt.Sprintf("%v", val)) - if err != nil { - log.Error().Msgf("Unable to read \"%s\" value from config", f.Name) - } else { - log.Debug().Msgf("Set parameter \"%s\" from configuration", f.Name) - } - } - }) -} diff --git a/programs/diagnostics/cmd/convert.go b/programs/diagnostics/cmd/convert.go deleted file mode 100644 index 1d619dd05e2..00000000000 --- a/programs/diagnostics/cmd/convert.go +++ /dev/null @@ -1 +0,0 @@ -package cmd diff --git a/programs/diagnostics/cmd/help.go b/programs/diagnostics/cmd/help.go deleted file mode 100644 index 750576dda25..00000000000 --- a/programs/diagnostics/cmd/help.go +++ /dev/null @@ -1,124 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" -) - -var cHelp = params.StringOptionsVar{ - Options: collectors.GetCollectorNames(false), - Value: "", -} -var oHelp = params.StringOptionsVar{ - Options: outputs.GetOutputNames(), - Value: "", -} - -func init() { - helpCmd.Flags().VarP(&cHelp, "collector", "c", "Specify collector to get description of available flags") - helpCmd.Flags().VarP(&oHelp, "output", "o", "Specify output to get description of available flags") - helpCmd.SetUsageTemplate(`Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}} - -Available Commands:{{range .Parent.Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} - -Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}} - -Alternatively use "{{.CommandPath}} [command] --help" for more information about a command. -`) - helpCmd.SetFlagErrorFunc(handleFlagErrors) - -} - -var helpCmd = &cobra.Command{ - Use: "help [command]", - Short: "Help about any command, collector or output", - Long: `Help provides help for any command, collector or output in the application.`, - Example: fmt.Sprintf(`%[1]v help collect -%[1]v help --collector=config -%[1]v help --output=simple`, rootCmd.Name()), - Run: func(c *cobra.Command, args []string) { - if len(args) != 0 { - //find the command on which help is requested - cmd, _, e := c.Root().Find(args) - if cmd == nil || e != nil { - c.Printf("Unknown help topic %#q\n", args) - cobra.CheckErr(c.Root().Usage()) - } else { - cmd.InitDefaultHelpFlag() - cobra.CheckErr(cmd.Help()) - } - return - } - if cHelp.Value != "" && oHelp.Value != "" { - log.Error().Msg("Specify either --collector or --output not both") - _ = c.Help() - os.Exit(1) - } - if cHelp.Value != "" { - collector, err := collectors.GetCollectorByName(cHelp.Value) - if err != nil { - log.Fatal().Err(err).Msgf("Unable to initialize collector %s", cHelp.Value) - } - configHelp(collector.Configuration(), "collector", cHelp.Value, collector.Description()) - } else if oHelp.Value != "" { - output, err := outputs.GetOutputByName(oHelp.Value) - if err != nil { - log.Fatal().Err(err).Msgf("Unable to initialize output %s", oHelp.Value) - } - configHelp(output.Configuration(), "output", oHelp.Value, output.Description()) - } else { - _ = c.Help() - } - os.Exit(0) - }, -} - -func configHelp(conf config.Configuration, componentType, name, description string) { - - paramMap := params.NewParamMap(map[string]config.Configuration{ - name: conf, - }) - tempHelpCmd := &cobra.Command{ - Use: fmt.Sprintf("--%s=%s", componentType, name), - Short: fmt.Sprintf("Help about the %s collector", name), - Long: description, - SilenceErrors: true, - Run: func(c *cobra.Command, args []string) { - _ = c.Help() - }, - } - params.AddParamMapToCmd(paramMap, tempHelpCmd, componentType, false) - // this is workaround to hide the help flag - tempHelpCmd.Flags().BoolP("help", "h", false, "Dummy help") - tempHelpCmd.Flags().Lookup("help").Hidden = true - tempHelpCmd.SetUsageTemplate(` -{{.Long}} - -Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}} - -Flags:{{if .HasAvailableLocalFlags}} -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{else}} - - No configuration flags available -{{end}} -`) - - _ = tempHelpCmd.Execute() -} diff --git a/programs/diagnostics/cmd/params/params.go b/programs/diagnostics/cmd/params/params.go deleted file mode 100644 index c4464aab5d2..00000000000 --- a/programs/diagnostics/cmd/params/params.go +++ /dev/null @@ -1,281 +0,0 @@ -package params - -import ( - "bytes" - "encoding/csv" - "fmt" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/spf13/cobra" -) - -type cliParamType uint8 - -const ( - String cliParamType = iota - StringList - StringOptionsList - Integer - Boolean -) - -type CliParam struct { - Description string - Default interface{} - //this should always be an address to a value - as required by cobra - Value interface{} - Type cliParamType -} - -type ParamMap map[string]map[string]CliParam - -func NewParamMap(configs map[string]config.Configuration) ParamMap { - paramMap := make(ParamMap) - for name, configuration := range configs { - for _, param := range configuration.Params { - switch p := param.(type) { - case config.StringParam: - paramMap = paramMap.createStringParam(name, p) - case config.StringListParam: - paramMap = paramMap.createStringListParam(name, p) - case config.StringOptions: - paramMap = paramMap.createStringOptionsParam(name, p) - case config.IntParam: - paramMap = paramMap.createIntegerParam(name, p) - case config.BoolParam: - paramMap = paramMap.createBoolParam(name, p) - } - } - } - return paramMap -} - -func (m ParamMap) createBoolParam(rootKey string, bParam config.BoolParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value bool - param := CliParam{ - Description: bParam.Description(), - Default: bParam.Value, - Value: &value, - Type: Boolean, - } - m[rootKey][bParam.Name()] = param - return m -} - -func (m ParamMap) createStringParam(rootKey string, sParam config.StringParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value string - param := CliParam{ - Description: sParam.Description(), - Default: sParam.Value, - Value: &value, - Type: String, - } - m[rootKey][sParam.Name()] = param - return m -} - -func (m ParamMap) createStringListParam(rootKey string, lParam config.StringListParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value []string - param := CliParam{ - Description: lParam.Description(), - Default: lParam.Values, - Value: &value, - Type: StringList, - } - m[rootKey][lParam.Name()] = param - return m -} - -func (m ParamMap) createStringOptionsParam(rootKey string, oParam config.StringOptions) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - value := StringOptionsVar{ - Options: oParam.Options, - Value: oParam.Value, - } - param := CliParam{ - Description: oParam.Description(), - Default: oParam.Value, - Value: &value, - Type: StringOptionsList, - } - m[rootKey][oParam.Name()] = param - return m -} - -func (m ParamMap) createIntegerParam(rootKey string, iParam config.IntParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value int64 - param := CliParam{ - Description: iParam.Description(), - Default: iParam.Value, - Value: &value, - Type: Integer, - } - m[rootKey][iParam.Name()] = param - return m -} - -func (c CliParam) GetConfigParam(name string) config.ConfigParam { - // this is a config being passed to a collector - required can be false - param := config.NewParam(name, c.Description, false) - switch c.Type { - case String: - return config.StringParam{ - Param: param, - // values will be pointers - Value: *(c.Value.(*string)), - } - case StringList: - return config.StringListParam{ - Param: param, - Values: *(c.Value.(*[]string)), - } - case StringOptionsList: - optionsVar := *(c.Value.(*StringOptionsVar)) - return config.StringOptions{ - Param: param, - Options: optionsVar.Options, - Value: optionsVar.Value, - } - case Integer: - return config.IntParam{ - Param: param, - Value: *(c.Value.(*int64)), - } - case Boolean: - return config.BoolParam{ - Param: param, - Value: *(c.Value.(*bool)), - } - } - return param -} - -type StringOptionsVar struct { - Options []string - Value string -} - -func (o StringOptionsVar) String() string { - return o.Value -} - -func (o *StringOptionsVar) Set(p string) error { - isIncluded := func(opts []string, val string) bool { - for _, opt := range opts { - if val == opt { - return true - } - } - return false - } - if !isIncluded(o.Options, p) { - return fmt.Errorf("%s is not included in options: %v", p, o.Options) - } - o.Value = p - return nil -} - -func (o *StringOptionsVar) Type() string { - return "string" -} - -type StringSliceOptionsVar struct { - Options []string - Values []string -} - -func (o StringSliceOptionsVar) String() string { - str, _ := writeAsCSV(o.Values) - return "[" + str + "]" -} - -func (o *StringSliceOptionsVar) Set(val string) error { - values, err := readAsCSV(val) - if err != nil { - return err - } - vValues := utils.Distinct(values, o.Options) - if len(vValues) > 0 { - return fmt.Errorf("%v are not included in options: %v", vValues, o.Options) - } - o.Values = values - return nil -} - -func (o *StringSliceOptionsVar) Type() string { - return "stringSlice" -} - -func writeAsCSV(vals []string) (string, error) { - b := &bytes.Buffer{} - w := csv.NewWriter(b) - err := w.Write(vals) - if err != nil { - return "", err - } - w.Flush() - return strings.TrimSuffix(b.String(), "\n"), nil -} - -func readAsCSV(val string) ([]string, error) { - if val == "" { - return []string{}, nil - } - stringReader := strings.NewReader(val) - csvReader := csv.NewReader(stringReader) - return csvReader.Read() -} - -func AddParamMapToCmd(paramMap ParamMap, cmd *cobra.Command, prefix string, hide bool) { - for rootKey, childMap := range paramMap { - for childKey, value := range childMap { - paramName := fmt.Sprintf("%s.%s.%s", prefix, rootKey, childKey) - switch value.Type { - case String: - cmd.Flags().StringVar(value.Value.(*string), paramName, value.Default.(string), value.Description) - case StringList: - cmd.Flags().StringSliceVar(value.Value.(*[]string), paramName, value.Default.([]string), value.Description) - case StringOptionsList: - cmd.Flags().Var(value.Value.(*StringOptionsVar), paramName, value.Description) - case Integer: - cmd.Flags().Int64Var(value.Value.(*int64), paramName, value.Default.(int64), value.Description) - case Boolean: - cmd.Flags().BoolVar(value.Value.(*bool), paramName, value.Default.(bool), value.Description) - } - // this ensures flags from collectors and outputs are not shown as they will pollute the output - if hide { - _ = cmd.Flags().MarkHidden(paramName) - } - } - } -} - -func ConvertParamsToConfig(paramMap ParamMap) map[string]config.Configuration { - configuration := make(map[string]config.Configuration) - for rootKey, childMap := range paramMap { - if _, ok := configuration[rootKey]; !ok { - configuration[rootKey] = config.Configuration{} - } - for childKey, value := range childMap { - configParam := value.GetConfigParam(childKey) - configuration[rootKey] = config.Configuration{Params: append(configuration[rootKey].Params, configParam)} - } - } - return configuration -} diff --git a/programs/diagnostics/cmd/params/params_test.go b/programs/diagnostics/cmd/params/params_test.go deleted file mode 100644 index 7671506ba59..00000000000 --- a/programs/diagnostics/cmd/params/params_test.go +++ /dev/null @@ -1,247 +0,0 @@ -package params_test - -import ( - "os" - "sort" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/spf13/cobra" - "github.com/stretchr/testify/require" -) - -var conf = map[string]config.Configuration{ - "config": { - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("directory", "A directory", false), - AllowEmpty: true, - }, - }, - }, - "system": { - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Include tables", false), - }, - config.StringListParam{ - Values: []string{"distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper"}, - Param: config.NewParam("exclude_tables", "Excluded tables", false), - }, - config.IntParam{ - Value: 100000, - Param: config.NewParam("row_limit", "Max rows", false), - }, - }, - }, - "reader": { - Params: []config.ConfigParam{ - config.StringOptions{ - Value: "csv", - Options: []string{"csv"}, - Param: config.NewParam("format", "Format of imported files", false), - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("collect_archives", "Collect archives", false), - }, - }, - }, -} - -func TestNewParamMap(t *testing.T) { - // test each of the types via NewParamMap - one with each type. the keys here can represent anything e.g. a collector name - t.Run("test param map correctly converts types", func(t *testing.T) { - paramMap := params.NewParamMap(conf) - require.Len(t, paramMap, 3) - // check config - require.Contains(t, paramMap, "config") - require.Len(t, paramMap["config"], 1) - require.Contains(t, paramMap["config"], "directory") - require.IsType(t, params.CliParam{}, paramMap["config"]["directory"]) - require.Equal(t, "A directory", paramMap["config"]["directory"].Description) - require.Equal(t, "", *(paramMap["config"]["directory"].Value.(*string))) - require.Equal(t, "", paramMap["config"]["directory"].Default) - require.Equal(t, params.String, paramMap["config"]["directory"].Type) - // check system - require.Contains(t, paramMap, "system") - require.Len(t, paramMap["system"], 3) - require.IsType(t, params.CliParam{}, paramMap["system"]["include_tables"]) - - require.Equal(t, "Include tables", paramMap["system"]["include_tables"].Description) - var value []string - require.Equal(t, &value, paramMap["system"]["include_tables"].Value) - require.Equal(t, value, paramMap["system"]["include_tables"].Default) - require.Equal(t, params.StringList, paramMap["system"]["include_tables"].Type) - - require.Equal(t, "Excluded tables", paramMap["system"]["exclude_tables"].Description) - require.IsType(t, params.CliParam{}, paramMap["system"]["exclude_tables"]) - require.Equal(t, &value, paramMap["system"]["exclude_tables"].Value) - require.Equal(t, []string{"distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper"}, paramMap["system"]["exclude_tables"].Default) - require.Equal(t, params.StringList, paramMap["system"]["exclude_tables"].Type) - - require.Equal(t, "Max rows", paramMap["system"]["row_limit"].Description) - require.IsType(t, params.CliParam{}, paramMap["system"]["row_limit"]) - var iValue int64 - require.Equal(t, &iValue, paramMap["system"]["row_limit"].Value) - require.Equal(t, int64(100000), paramMap["system"]["row_limit"].Default) - require.Equal(t, params.Integer, paramMap["system"]["row_limit"].Type) - - // check reader - require.Contains(t, paramMap, "reader") - require.Len(t, paramMap["reader"], 2) - require.IsType(t, params.CliParam{}, paramMap["reader"]["format"]) - require.Equal(t, "Format of imported files", paramMap["reader"]["format"].Description) - require.IsType(t, params.CliParam{}, paramMap["reader"]["format"]) - oValue := params.StringOptionsVar{ - Options: []string{"csv"}, - Value: "csv", - } - require.Equal(t, &oValue, paramMap["reader"]["format"].Value) - require.Equal(t, "csv", paramMap["reader"]["format"].Default) - require.Equal(t, params.StringOptionsList, paramMap["reader"]["format"].Type) - - require.IsType(t, params.CliParam{}, paramMap["reader"]["collect_archives"]) - require.Equal(t, "Collect archives", paramMap["reader"]["collect_archives"].Description) - require.IsType(t, params.CliParam{}, paramMap["reader"]["collect_archives"]) - var bVar bool - require.Equal(t, &bVar, paramMap["reader"]["collect_archives"].Value) - require.Equal(t, true, paramMap["reader"]["collect_archives"].Default) - require.Equal(t, params.Boolean, paramMap["reader"]["collect_archives"].Type) - - }) - -} - -// test GetConfigParam -func TestConvertParamsToConfig(t *testing.T) { - paramMap := params.NewParamMap(conf) - t.Run("test we can convert a param map back to a config", func(t *testing.T) { - cParam := params.ConvertParamsToConfig(paramMap) - // these will not be equal as we have some information loss e.g. allowEmpty - //require.Equal(t, conf, cParam) - // deep equality - for name := range conf { - require.Equal(t, len(conf[name].Params), len(cParam[name].Params)) - // sort both consistently - sort.Slice(conf[name].Params, func(i, j int) bool { - return conf[name].Params[i].Name() < conf[name].Params[j].Name() - }) - sort.Slice(cParam[name].Params, func(i, j int) bool { - return cParam[name].Params[i].Name() < cParam[name].Params[j].Name() - }) - for i, param := range conf[name].Params { - require.Equal(t, param.Required(), cParam[name].Params[i].Required()) - require.Equal(t, param.Name(), cParam[name].Params[i].Name()) - require.Equal(t, param.Description(), cParam[name].Params[i].Description()) - } - } - }) -} - -// create via NewParamMap and add to command AddParamMapToCmd - check contents -func TestAddParamMapToCmd(t *testing.T) { - paramMap := params.NewParamMap(conf) - t.Run("test we can add hidden params to a command", func(t *testing.T) { - testComand := &cobra.Command{ - Use: "test", - Short: "Run a test", - Long: `Longer description`, - Run: func(cmd *cobra.Command, args []string) { - os.Exit(0) - }, - } - params.AddParamMapToCmd(paramMap, testComand, "collector", true) - // check we get an error on one which doesn't exist - _, err := testComand.Flags().GetString("collector.config.random") - require.NotNil(t, err) - // check getting incorrect type - _, err = testComand.Flags().GetString("collector.system.include_tables") - require.NotNil(t, err) - - // check existence of all flags - directory, err := testComand.Flags().GetString("collector.config.directory") - require.Nil(t, err) - require.Equal(t, "", directory) - - includeTables, err := testComand.Flags().GetStringSlice("collector.system.include_tables") - require.Nil(t, err) - require.Equal(t, []string{}, includeTables) - - excludeTables, err := testComand.Flags().GetStringSlice("collector.system.exclude_tables") - require.Nil(t, err) - require.Equal(t, []string{"distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper"}, excludeTables) - - rowLimit, err := testComand.Flags().GetInt64("collector.system.row_limit") - require.Nil(t, err) - require.Equal(t, int64(100000), rowLimit) - - format, err := testComand.Flags().GetString("collector.reader.format") - require.Nil(t, err) - require.Equal(t, "csv", format) - - collectArchives, err := testComand.Flags().GetBool("collector.reader.collect_archives") - require.Nil(t, err) - require.Equal(t, true, collectArchives) - }) -} - -// test StringOptionsVar -func TestStringOptionsVar(t *testing.T) { - - t.Run("test we can set", func(t *testing.T) { - format := params.StringOptionsVar{ - Options: []string{"csv", "tsv", "native"}, - Value: "csv", - } - require.Equal(t, "csv", format.String()) - err := format.Set("tsv") - require.Nil(t, err) - require.Equal(t, "tsv", format.String()) - }) - - t.Run("test set invalid", func(t *testing.T) { - format := params.StringOptionsVar{ - Options: []string{"csv", "tsv", "native"}, - Value: "csv", - } - require.Equal(t, "csv", format.String()) - err := format.Set("random") - require.NotNil(t, err) - require.Equal(t, "random is not included in options: [csv tsv native]", err.Error()) - }) -} - -// test StringSliceOptionsVar -func TestStringSliceOptionsVar(t *testing.T) { - - t.Run("test we can set", func(t *testing.T) { - formats := params.StringSliceOptionsVar{ - Options: []string{"csv", "tsv", "native", "qsv"}, - Values: []string{"csv", "tsv"}, - } - require.Equal(t, "[csv,tsv]", formats.String()) - err := formats.Set("tsv,native") - require.Nil(t, err) - require.Equal(t, "[tsv,native]", formats.String()) - }) - - t.Run("test set invalid", func(t *testing.T) { - formats := params.StringSliceOptionsVar{ - Options: []string{"csv", "tsv", "native", "qsv"}, - Values: []string{"csv", "tsv"}, - } - require.Equal(t, "[csv,tsv]", formats.String()) - err := formats.Set("tsv,random") - require.NotNil(t, err) - require.Equal(t, "[random] are not included in options: [csv tsv native qsv]", err.Error()) - err = formats.Set("msv,random") - require.NotNil(t, err) - require.Equal(t, "[msv random] are not included in options: [csv tsv native qsv]", err.Error()) - }) - -} diff --git a/programs/diagnostics/cmd/root.go b/programs/diagnostics/cmd/root.go deleted file mode 100644 index 4cf329d5438..00000000000 --- a/programs/diagnostics/cmd/root.go +++ /dev/null @@ -1,174 +0,0 @@ -package cmd - -import ( - "fmt" - "net/http" - _ "net/http/pprof" - "os" - "strings" - "time" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func enableDebug() { - if debug { - zerolog.SetGlobalLevel(zerolog.DebugLevel) - go func() { - err := http.ListenAndServe("localhost:8080", nil) - if err != nil { - log.Error().Err(err).Msg("unable to start debugger") - } else { - log.Debug().Msg("debugger has been started on port 8080") - } - }() - } -} - -var rootCmd = &cobra.Command{ - Use: "clickhouse-diagnostics", - Short: "Capture and convert ClickHouse diagnostic bundles.", - Long: `Captures ClickHouse diagnostic bundles to a number of supported formats, including file and ClickHouse itself. Converts bundles between formats.`, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - enableDebug() - err := initializeConfig() - if err != nil { - log.Error().Err(err) - os.Exit(1) - } - return nil - }, - Example: `clickhouse-diagnostics collect`, -} - -const ( - colorRed = iota + 31 - colorGreen - colorYellow - colorMagenta = 35 - - colorBold = 1 -) - -const TimeFormat = time.RFC3339 - -var debug bool -var configFiles []string - -const ( - // The environment variable prefix of all environment variables bound to our command line flags. - // For example, --output is bound to CLICKHOUSE_DIAGNOSTIC_OUTPUT. - envPrefix = "CLICKHOUSE_DIAGNOSTIC" -) - -func init() { - rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Enable debug mode") - rootCmd.PersistentFlags().StringSliceVarP(&configFiles, "config", "f", []string{"clickhouse-diagnostics.yml", "/etc/clickhouse-diagnostics.yml"}, "Configuration file path") - // set a usage template to ensure flags on root are listed as global - rootCmd.SetUsageTemplate(`Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} - {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} - -Aliases: - {{.NameAndAliases}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}}{{if .HasAvailableSubCommands}} - -Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} - -Global Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} - -Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} - {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} - -Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}} -`) - rootCmd.SetFlagErrorFunc(handleFlagErrors) - -} - -func Execute() { - // logs go to stderr - stdout is exclusive for outputs e.g. tables - output := zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: TimeFormat} - // override the colors - output.FormatLevel = func(i interface{}) string { - var l string - if ll, ok := i.(string); ok { - switch ll { - case zerolog.LevelTraceValue: - l = colorize("TRC", colorMagenta) - case zerolog.LevelDebugValue: - l = colorize("DBG", colorMagenta) - case zerolog.LevelInfoValue: - l = colorize("INF", colorGreen) - case zerolog.LevelWarnValue: - l = colorize(colorize("WRN", colorYellow), colorBold) - case zerolog.LevelErrorValue: - l = colorize(colorize("ERR", colorRed), colorBold) - case zerolog.LevelFatalValue: - l = colorize(colorize("FTL", colorRed), colorBold) - case zerolog.LevelPanicValue: - l = colorize(colorize("PNC", colorRed), colorBold) - default: - l = colorize("???", colorBold) - } - } else { - if i == nil { - l = colorize("???", colorBold) - } else { - l = strings.ToUpper(fmt.Sprintf("%s", i))[0:3] - } - } - return l - } - output.FormatTimestamp = func(i interface{}) string { - tt := i.(string) - return colorize(tt, colorGreen) - } - log.Logger = log.Output(output) - zerolog.SetGlobalLevel(zerolog.InfoLevel) - rootCmd.SetHelpCommand(helpCmd) - if err := rootCmd.Execute(); err != nil { - log.Fatal().Err(err) - } -} - -// colorize returns the string s wrapped in ANSI code c -func colorize(s interface{}, c int) string { - return fmt.Sprintf("\x1b[%dm%v\x1b[0m", c, s) -} - -func handleFlagErrors(cmd *cobra.Command, err error) error { - fmt.Println(colorize(colorize(fmt.Sprintf("Error: %s\n", err), colorRed), colorBold)) - _ = cmd.Help() - os.Exit(1) - return nil -} - -func initializeConfig() error { - // we use the first config file we find - var configFile string - for _, confFile := range configFiles { - if ok, _ := utils.FileExists(confFile); ok { - configFile = confFile - break - } - } - if configFile == "" { - log.Warn().Msgf("config file in %s not found - config file will be ignored", configFiles) - return nil - } - viper.SetConfigFile(configFile) - if err := viper.ReadInConfig(); err != nil { - return errors.Wrapf(err, "Unable to read configuration file at %s", configFile) - } - return nil -} diff --git a/programs/diagnostics/cmd/version.go b/programs/diagnostics/cmd/version.go deleted file mode 100644 index b1c0b44171b..00000000000 --- a/programs/diagnostics/cmd/version.go +++ /dev/null @@ -1,24 +0,0 @@ -package cmd - -import ( - "fmt" - "github.com/spf13/cobra" -) - -var ( - Version = "" // set at compile time with -ldflags "-X versserv/cmd.Version=x.y.yz" - Commit = "" -) - -func init() { - rootCmd.AddCommand(versionCmd) -} - -var versionCmd = &cobra.Command{ - Use: "version", - Short: "Print the version number of clickhouse-diagnostics", - Long: `All software has versions. This is clickhouse-diagnostics`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Clickhouse Diagnostics %s (%s)\n", Version, Commit) - }, -} diff --git a/programs/diagnostics/go.mod b/programs/diagnostics/go.mod deleted file mode 100644 index 34c6b0037ae..00000000000 --- a/programs/diagnostics/go.mod +++ /dev/null @@ -1,89 +0,0 @@ -module github.com/ClickHouse/ClickHouse/programs/diagnostics - -go 1.19 - -require ( - github.com/ClickHouse/clickhouse-go/v2 v2.0.12 - github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/Masterminds/semver v1.5.0 - github.com/bmatcuk/doublestar/v4 v4.0.2 - github.com/docker/go-connections v0.4.0 - github.com/elastic/gosigar v0.14.2 - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/jaypipes/ghw v0.8.0 - github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f - github.com/mholt/archiver/v4 v4.0.0-alpha.4 - github.com/olekukonko/tablewriter v0.0.5 - github.com/pkg/errors v0.9.1 - github.com/rs/zerolog v1.26.1 - github.com/spf13/cobra v1.3.0 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.10.1 - github.com/stretchr/testify v1.8.1 - github.com/testcontainers/testcontainers-go v0.18.0 - github.com/yargevad/filepathx v1.0.0 - gopkg.in/yaml.v3 v3.0.1 -) - -require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect - github.com/andybalholm/brotli v1.0.4 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect - github.com/containerd/containerd v1.6.17 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/distribution v2.8.2+incompatible // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/docker v23.0.0+incompatible // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/dsnet/compress v0.0.1 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-ole/go-ole v1.2.4 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jaypipes/pcidb v0.6.0 // indirect - github.com/klauspost/compress v1.13.6 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect - github.com/moby/patternmatcher v0.5.0 // indirect - github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2 // indirect - github.com/opencontainers/runc v1.1.3 // indirect - github.com/paulmach/orb v0.4.0 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pierrec/lz4/v4 v4.1.14 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect - github.com/spf13/afero v1.8.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect - github.com/therootcompany/xz v1.0.1 // indirect - github.com/ulikunitz/xz v0.5.10 // indirect - go.opentelemetry.io/otel v1.4.1 // indirect - go.opentelemetry.io/otel/trace v1.4.1 // indirect - golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad // indirect - google.golang.org/grpc v1.47.0 // indirect - google.golang.org/protobuf v1.28.0 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect -) diff --git a/programs/diagnostics/go.sum b/programs/diagnostics/go.sum deleted file mode 100644 index a95dfb4fd2b..00000000000 --- a/programs/diagnostics/go.sum +++ /dev/null @@ -1,992 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/clickhouse-go v1.5.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/ClickHouse/clickhouse-go/v2 v2.0.12 h1:Nbl/NZwoM6LGJm7smNBgvtdr/rxjlIssSW3eG/Nmb9E= -github.com/ClickHouse/clickhouse-go/v2 v2.0.12/go.mod h1:u4RoNQLLM2W6hNSPYrIESLJqaWSInZVmfM+MlaAhXcg= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.9.6 h1:VwnDOgLeoi2du6dAznfmspNqTiwczvjv4K7NxuY9jsY= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= -github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= -github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.6.17 h1:XDnJIeJW0cLf6v7/+N+6L9kGrChHeXekZp2VHu6OpiY= -github.com/containerd/containerd v1.6.17/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/distribution v2.8.2+incompatible h1:k9+4DKdOG+quPFZXT/mUsiQrGu9vYCp+dXpuPkuqhk8= -github.com/distribution/distribution v2.8.2+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v23.0.0+incompatible h1:L6c28tNyqZ4/ub9AZC9d5QUuunoHHfEH4/Ue+h/E5nE= -github.com/docker/docker v23.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jaypipes/ghw v0.8.0 h1:02q1pTm9CD83vuhBsEZZhOCS128pq87uyaQeJZkp3sQ= -github.com/jaypipes/ghw v0.8.0/go.mod h1:+gR9bjm3W/HnFi90liF+Fj9GpCe/Dsibl9Im8KmC7c4= -github.com/jaypipes/pcidb v0.6.0 h1:VIM7GKVaW4qba30cvB67xSCgJPTzkG8Kzw/cbs5PHWU= -github.com/jaypipes/pcidb v0.6.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f h1:B0OD7nYl2FPQEVrw8g2uyc1lGEzNbvrKh7fspGZcbvY= -github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f/go.mod h1:aEt7p9Rvh67BYApmZwNDPpgircTO2kgdmDUoF/1QmwA= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mholt/archiver/v4 v4.0.0-alpha.4 h1:QJ4UuWgavPynEX3LXxClHDRGzYcgcvTtAMp8az7spuw= -github.com/mholt/archiver/v4 v4.0.0-alpha.4/go.mod h1:J7SYS/UTAtnO3I49RQEf+2FYZVwo7XBOh9Im43VrjNs= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= -github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= -github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f h1:J/7hjLaHLD7epG0m6TBMGmp4NQ+ibBYLfeyJWdAIFLA= -github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f/go.mod h1:15ce4BGCFxt7I5NQKT+HV0yEDxmf6fSysfEDiVo3zFM= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= -github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= -github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= -github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/paulmach/orb v0.4.0 h1:ilp1MQjRapLJ1+qcays1nZpe0mvkCY+b8JU/qBKRZ1A= -github.com/paulmach/orb v0.4.0/go.mod h1:FkcWtplUAIVqAuhAOV2d3rpbnQyliDOjOcLW9dUrfdU= -github.com/paulmach/protoscan v0.2.1-0.20210522164731-4e53c6875432/go.mod h1:2sV+uZ/oQh66m4XJVZm5iqUZ62BN88Ex1E+TTS0nLzI= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= -github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= -github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60= -github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= -github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= -github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= -github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/testcontainers/testcontainers-go v0.18.0 h1:8RXrcIQv5xX/uBOSmZd297gzvA7F0yuRA37/918o7Yg= -github.com/testcontainers/testcontainers-go v0.18.0/go.mod h1:rLC7hR2SWRjJZZNrUYiTKvUXCziNxzZiYtz9icTWYNQ= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= -github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= -github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/otel v1.4.1 h1:QbINgGDDcoQUoMJa2mMaWno49lja9sHwp6aoa2n3a4g= -go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= -go.opentelemetry.io/otel/trace v1.4.1 h1:O+16qcdTrT7zxv2J6GejTPFinSwA++cYerC5iSiF8EQ= -go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad h1:kqrS+lhvaMHCxul6sKQvKJ8nAAhlVItmZV822hYFH/U= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= -howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/programs/diagnostics/internal/collectors/clickhouse/config.go b/programs/diagnostics/internal/collectors/clickhouse/config.go deleted file mode 100644 index 92368bce6f3..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/config.go +++ /dev/null @@ -1,113 +0,0 @@ -package clickhouse - -import ( - "fmt" - "path/filepath" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" -) - -type ConfigCollector struct { - resourceManager *platform.ResourceManager -} - -func NewConfigCollector(m *platform.ResourceManager) *ConfigCollector { - return &ConfigCollector{ - resourceManager: m, - } -} - -const DefaultConfigLocation = "/etc/clickhouse-server/" -const ProcessedConfigurationLocation = "/var/lib/clickhouse/preprocessed_configs" - -func (c ConfigCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(c.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - directory, err := config.ReadStringValue(conf, "directory") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - if directory != "" { - // user has specified a directory - we therefore skip all other efforts to locate the config - frame, errs := data.NewConfigFileFrame(directory) - return &data.DiagnosticBundle{ - Frames: map[string]data.Frame{ - "user_specified": frame, - }, - Errors: data.FrameErrors{Errors: errs}, - }, nil - } - configCandidates, err := FindConfigurationFiles() - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "Unable to find configuration files") - } - frames := make(map[string]data.Frame) - var frameErrors []error - for frameName, confDir := range configCandidates { - frame, errs := data.NewConfigFileFrame(confDir) - frameErrors = append(frameErrors, errs...) - frames[frameName] = frame - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: data.FrameErrors{Errors: frameErrors}, - }, err -} - -func FindConfigurationFiles() (map[string]string, error) { - configCandidates := map[string]string{ - "default": DefaultConfigLocation, - "preprocessed": ProcessedConfigurationLocation, - } - // we don't know specifically where the config is but try to find via processes - processConfigs, err := utils.FindConfigsFromClickHouseProcesses() - if err != nil { - return nil, err - } - for i, path := range processConfigs { - confDir := filepath.Dir(path) - if len(processConfigs) == 1 { - configCandidates["process"] = confDir - break - } - configCandidates[fmt.Sprintf("process_%d", i)] = confDir - } - return configCandidates, nil -} - -func (c ConfigCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("directory", "Specify the location of the configuration files for ClickHouse Server e.g. /etc/clickhouse-server/", false), - AllowEmpty: true, - }, - }, - } -} - -func (c ConfigCollector) Description() string { - return "Collects the ClickHouse configuration from the local filesystem." -} - -func (c ConfigCollector) IsDefault() bool { - return true -} - -// here we register the collector for use -func init() { - collectors.Register("config", func() (collectors.Collector, error) { - return &ConfigCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/config_test.go b/programs/diagnostics/internal/collectors/clickhouse/config_test.go deleted file mode 100644 index 355cbb65620..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/config_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package clickhouse_test - -import ( - "encoding/xml" - "fmt" - "io" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestConfigConfiguration(t *testing.T) { - t.Run("correct configuration is returned for config collector", func(t *testing.T) { - configCollector := clickhouse.NewConfigCollector(&platform.ResourceManager{}) - conf := configCollector.Configuration() - require.Len(t, conf.Params, 1) - // check first param - require.IsType(t, config.StringParam{}, conf.Params[0]) - directory, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, directory.Required()) - require.Equal(t, directory.Name(), "directory") - require.Equal(t, "", directory.Value) - }) -} - -func TestConfigCollect(t *testing.T) { - configCollector := clickhouse.NewConfigCollector(&platform.ResourceManager{}) - - t.Run("test default file collector configuration", func(t *testing.T) { - diagSet, err := configCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - // we won't be able to collect the default configs preprocessed and default - even if clickhouse is installed - // these directories should not be readable under any permissions these tests are unrealistically executed! - // note: we may also pick up configs from a local clickhouse process - we thus allow a len >=2 but don't check this - // as its non-deterministic - require.GreaterOrEqual(t, len(diagSet.Frames), 2) - // check default key - require.Contains(t, diagSet.Frames, "default") - require.Equal(t, diagSet.Frames["default"].Name(), "/etc/clickhouse-server/") - require.Equal(t, diagSet.Frames["default"].Columns(), []string{"config"}) - // collection will have failed - checkFrame(t, diagSet.Frames["default"], nil) - // check preprocessed key - require.Contains(t, diagSet.Frames, "preprocessed") - require.Equal(t, diagSet.Frames["preprocessed"].Name(), "/var/lib/clickhouse/preprocessed_configs") - require.Equal(t, diagSet.Frames["preprocessed"].Columns(), []string{"config"}) - // min of 2 - might be more if a local installation of clickhouse is running - require.GreaterOrEqual(t, len(diagSet.Errors.Errors), 2) - }) - - t.Run("test configuration when specified", func(t *testing.T) { - // create some test files - tempDir := t.TempDir() - confDir := path.Join(tempDir, "conf") - // create an includes file - includesDir := path.Join(tempDir, "includes") - err := os.MkdirAll(includesDir, os.ModePerm) - require.Nil(t, err) - includesPath := path.Join(includesDir, "random.xml") - includesFile, err := os.Create(includesPath) - require.Nil(t, err) - xmlWriter := io.Writer(includesFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - xmlConfig := data.XmlConfig{ - XMLName: xml.Name{}, - Clickhouse: data.XmlLoggerConfig{ - XMLName: xml.Name{}, - ErrorLog: "/var/log/clickhouse-server/clickhouse-server.err.log", - Log: "/var/log/clickhouse-server/clickhouse-server.log", - }, - IncludeFrom: "", - } - err = enc.Encode(xmlConfig) - require.Nil(t, err) - // create 5 temporary config files - length is 6 for the included file - rows := make([][]interface{}, 6) - for i := 0; i < 5; i++ { - if i == 4 { - // set the includes for the last doc - xmlConfig.IncludeFrom = includesPath - } - // we want to check hierarchies are walked so create a simple folder for each file - fileDir := path.Join(confDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.xml", i)) - row := make([]interface{}, 1) - row[0] = data.XmlConfigFile{Path: filepath} - rows[i] = row - xmlFile, err := os.Create(filepath) - require.Nil(t, err) - // write a little xml so its valid - xmlConfig := xmlConfig - xmlWriter := io.Writer(xmlFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - err = enc.Encode(xmlConfig) - require.Nil(t, err) - } - diagSet, err := configCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: confDir, - Param: config.NewParam("directory", "File locations", false), - }, - }, - }) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Frames, 1) - require.Contains(t, diagSet.Frames, "user_specified") - require.Equal(t, diagSet.Frames["user_specified"].Name(), confDir) - require.Equal(t, diagSet.Frames["user_specified"].Columns(), []string{"config"}) - iConf := make([]interface{}, 1) - iConf[0] = data.XmlConfigFile{Path: includesPath, Included: true} - rows[5] = iConf - checkFrame(t, diagSet.Frames["user_specified"], rows) - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/db_logs.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs.go deleted file mode 100644 index 3253f504c1b..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/db_logs.go +++ /dev/null @@ -1,108 +0,0 @@ -package clickhouse - -import ( - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" -) - -type DBLogTable struct { - orderBy data.OrderBy - excludeColumns []string -} - -var DbLogTables = map[string]DBLogTable{ - "query_log": { - orderBy: data.OrderBy{ - Column: "event_time_microseconds", - Order: data.Asc, - }, - excludeColumns: []string{}, - }, - "query_thread_log": { - orderBy: data.OrderBy{ - Column: "event_time_microseconds", - Order: data.Asc, - }, - excludeColumns: []string{}, - }, - "text_log": { - orderBy: data.OrderBy{ - Column: "event_time_microseconds", - Order: data.Asc, - }, - excludeColumns: []string{}, - }, -} - -// This collector collects db logs - -type DBLogsCollector struct { - resourceManager *platform.ResourceManager -} - -func NewDBLogsCollector(m *platform.ResourceManager) *DBLogsCollector { - return &DBLogsCollector{ - resourceManager: m, - } -} - -func (dc *DBLogsCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(dc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - frames := make(map[string]data.Frame) - var frameErrors []error - for logTable, tableConfig := range DbLogTables { - frame, err := dc.resourceManager.DbClient.ReadTable("system", logTable, tableConfig.excludeColumns, tableConfig.orderBy, rowLimit) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to collect %s", logTable)) - } else { - frames[logTable] = frame - } - } - - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -func (dc *DBLogsCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 100000, - Param: config.NewParam("row_limit", "Maximum number of log rows to collect. Negative values mean unlimited", false), - }, - }, - } -} - -func (dc *DBLogsCollector) IsDefault() bool { - return true -} - -func (dc DBLogsCollector) Description() string { - return "Collects the ClickHouse logs directly from the database." -} - -// here we register the collector for use -func init() { - collectors.Register("db_logs", func() (collectors.Collector, error) { - return &DBLogsCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go deleted file mode 100644 index 3fc585f3352..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestDbLogsConfiguration(t *testing.T) { - t.Run("correct configuration is returned for summary collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - dbLogsCollector := clickhouse.NewDBLogsCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := dbLogsCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.IntParam{}, conf.Params[0]) - rowLimit, ok := conf.Params[0].(config.IntParam) - require.True(t, ok) - require.False(t, rowLimit.Required()) - require.Equal(t, rowLimit.Name(), "row_limit") - require.Equal(t, int64(100000), rowLimit.Value) - }) -} - -func TestDbLogsCollect(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - dbLogsCollector := clickhouse.NewDBLogsCollector(&platform.ResourceManager{ - DbClient: client, - }) - queryLogColumns := []string{"type", "event_date", "event_time", "event_time_microseconds", - "query_start_time", "query_start_time_microseconds", "query_duration_ms", "read_rows", "read_bytes", "written_rows", "written_bytes", - "result_rows", "result_bytes", "memory_usage", "current_database", "query", "formatted_query", "normalized_query_hash", - "query_kind", "databases", "tables", "columns", "projections", "views", "exception_code", "exception", "stack_trace", - "is_initial_query", "user", "query_id", "address", "port", "initial_user", "initial_query_id", "initial_address", "initial_port", - "initial_query_start_time", "initial_query_start_time_microseconds", "interface", "os_user", "client_hostname", "client_name", - "client_revision", "client_version_major", "client_version_minor", "client_version_patch", "http_method", "http_user_agent", - "http_referer", "forwarded_for", "quota_key", "revision", "log_comment", "thread_ids", "ProfileEvents", "Settings", - "used_aggregate_functions", "used_aggregate_function_combinators", "used_database_engines", "used_data_type_families", - "used_dictionaries", "used_formats", "used_functions", "used_storages", "used_table_functions"} - queryLogFrame := test.NewFakeDataFrame("queryLog", queryLogColumns, - [][]interface{}{ - {"QueryStart", "2021-12-13", "2021-12-13 12:53:20", "2021-12-13 12:53:20.590579", "2021-12-13 12:53:20", "2021-12-13 12:53:20.590579", "0", "0", "0", "0", "0", "0", "0", "0", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "2021-12-13 12:53:30", "2021-12-13 12:53:30.590579", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[]", "{}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]"}, - {"QueryFinish", "2021-12-13", "2021-12-13 12:53:30", "2021-12-13 12:53:30.607292", "2021-12-13 12:53:30", "2021-12-13 12:53:30.590579", "15", "4512", "255694", "0", "0", "4358", "173248", "4415230", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "2021-12-13 12:53:30", "2021-12-13 12:53:30.590579", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[95298,95315,95587,95316,95312,95589,95318,95586,95588,95585]", "{'Query':1,'SelectQuery':1,'ArenaAllocChunks':41,'ArenaAllocBytes':401408,'FunctionExecute':62,'NetworkSendElapsedMicroseconds':463,'NetworkSendBytes':88452,'SelectedRows':4512,'SelectedBytes':255694,'RegexpCreated':6,'ContextLock':411,'RWLockAcquiredReadLocks':190,'RealTimeMicroseconds':49221,'UserTimeMicroseconds':19811,'SystemTimeMicroseconds':2817,'SoftPageFaults':1128,'OSCPUWaitMicroseconds':127,'OSCPUVirtualTimeMicroseconds':22624,'OSWriteBytes':12288,'OSWriteChars':13312}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "['concat','notEmpty','extractAll']", "[]", "[]"}, - {"QueryStart", "2021-12-13", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "0", "0", "0", "0", "0", "0", "0", "0", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[]", "{}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]"}, - {"QueryFinish", "2021-12-13", "2021-12-13 13:02:56", "2021-12-13 13:02:56.437115", "2021-12-13 13:02:56", "2021-12-13 13:02:56.419528", "16", "4629", "258376", "0", "0", "4377", "174272", "4404694", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[95298,95318,95315,95316,95312,95588,95589,95586,95585,95587]", "{'Query':1,'SelectQuery':1,'ArenaAllocChunks':41,'ArenaAllocBytes':401408,'FunctionExecute':62,'NetworkSendElapsedMicroseconds':740,'NetworkSendBytes':88794,'SelectedRows':4629,'SelectedBytes':258376,'ContextLock':411,'RWLockAcquiredReadLocks':194,'RealTimeMicroseconds':52469,'UserTimeMicroseconds':17179,'SystemTimeMicroseconds':4218,'SoftPageFaults':569,'OSCPUWaitMicroseconds':303,'OSCPUVirtualTimeMicroseconds':25087,'OSWriteBytes':12288,'OSWriteChars':12288}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "['concat','notEmpty','extractAll']", "[]", "[]"}, - }) - - client.QueryResponses["SELECT * FROM system.query_log ORDER BY event_time_microseconds ASC LIMIT 100000"] = &queryLogFrame - - textLogColumns := []string{"event_date", "event_time", "event_time_microseconds", "microseconds", "thread_name", "thread_id", "level", "query_id", "logger_name", "message", "revision", "source_file", "source_line"} - textLogFrame := test.NewFakeDataFrame("textLog", textLogColumns, - [][]interface{}{ - {"2022-02-03", "2022-02-03 16:17:47", "2022-02-03 16:37:17.056950", "56950", "clickhouse-serv", "68947", "Information", "", "DNSCacheUpdater", "Update period 15 seconds", "54458", "../src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start()", "46"}, - {"2022-02-03", "2022-02-03 16:27:47", "2022-02-03 16:37:27.057022", "57022", "clickhouse-serv", "68947", "Information", "", "Application", "Available RAM: 62.24 GiB; physical cores: 8; logical cores: 16.", "54458", "../programs/server/Server.cpp; virtual int DB::Server::main(const std::vector &)", "1380"}, - {"2022-02-03", "2022-02-03 16:37:47", "2022-02-03 16:37:37.057484", "57484", "clickhouse-serv", "68947", "Information", "", "Application", "Listening for http://[::1]:8123", "54458", "../programs/server/Server.cpp; virtual int DB::Server::main(const std::vector &)", "1444"}, - {"2022-02-03", "2022-02-03 16:47:47", "2022-02-03 16:37:47.057527", "57527", "clickhouse-serv", "68947", "Information", "", "Application", "Listening for native protocol (tcp): [::1]:9000", "54458", "../programs/server/Server.cpp; virtual int DB::Server::main(const std::vector &)", "1444"}, - }) - - client.QueryResponses["SELECT * FROM system.text_log ORDER BY event_time_microseconds ASC LIMIT 100000"] = &textLogFrame - - // skip query_thread_log frame - often it doesn't exist anyway unless enabled - t.Run("test default db logs collection", func(t *testing.T) { - bundle, errs := dbLogsCollector.Collect(config.Configuration{}) - require.Empty(t, errs) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 2) - require.Contains(t, bundle.Frames, "text_log") - require.Contains(t, bundle.Frames, "query_log") - require.Len(t, bundle.Errors.Errors, 1) - // check query_log frame - require.Contains(t, bundle.Frames, "query_log") - require.Equal(t, queryLogColumns, bundle.Frames["query_log"].Columns()) - checkFrame(t, bundle.Frames["query_log"], queryLogFrame.Rows) - //check text_log frame - require.Contains(t, bundle.Frames, "text_log") - require.Equal(t, textLogColumns, bundle.Frames["text_log"].Columns()) - checkFrame(t, bundle.Frames["text_log"], textLogFrame.Rows) - client.Reset() - }) - - t.Run("test db logs collection with limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 1, - Param: config.NewParam("row_limit", "Maximum number of log rows to collect. Negative values mean unlimited", false), - }, - }, - } - bundle, err := dbLogsCollector.Collect(conf) - require.Empty(t, err) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 0) - require.Len(t, bundle.Errors.Errors, 3) - // populate client - client.QueryResponses["SELECT * FROM system.query_log ORDER BY event_time_microseconds ASC LIMIT 1"] = &queryLogFrame - client.QueryResponses["SELECT * FROM system.text_log ORDER BY event_time_microseconds ASC LIMIT 1"] = &textLogFrame - bundle, err = dbLogsCollector.Collect(conf) - require.Empty(t, err) - require.Len(t, bundle.Frames, 2) - require.Len(t, bundle.Errors.Errors, 1) - require.Contains(t, bundle.Frames, "text_log") - require.Contains(t, bundle.Frames, "query_log") - // check query_log frame - require.Contains(t, bundle.Frames, "query_log") - require.Equal(t, queryLogColumns, bundle.Frames["query_log"].Columns()) - checkFrame(t, bundle.Frames["query_log"], queryLogFrame.Rows[:1]) - //check text_log frame - require.Contains(t, bundle.Frames, "text_log") - require.Equal(t, textLogColumns, bundle.Frames["text_log"].Columns()) - checkFrame(t, bundle.Frames["text_log"], textLogFrame.Rows[:1]) - client.Reset() - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/logs.go b/programs/diagnostics/internal/collectors/clickhouse/logs.go deleted file mode 100644 index 8436a392c47..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/logs.go +++ /dev/null @@ -1,140 +0,0 @@ -package clickhouse - -import ( - "fmt" - "path/filepath" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" -) - -// This collector collects logs - -type LogsCollector struct { - resourceManager *platform.ResourceManager -} - -func NewLogsCollector(m *platform.ResourceManager) *LogsCollector { - return &LogsCollector{ - resourceManager: m, - } -} - -var DefaultLogsLocation = filepath.Clean("/var/log/clickhouse-server/") - -func (lc *LogsCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(lc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - directory, err := config.ReadStringValue(conf, "directory") - if err != nil { - return &data.DiagnosticBundle{}, err - } - collectArchives, err := config.ReadBoolValue(conf, "collect_archives") - if err != nil { - return &data.DiagnosticBundle{}, err - } - logPatterns := []string{"*.log"} - if collectArchives { - logPatterns = append(logPatterns, "*.gz") - } - - if directory != "" { - // user has specified a directory - we therefore skip all other efforts to locate the logs - frame, errs := data.NewFileDirectoryFrame(directory, logPatterns) - return &data.DiagnosticBundle{ - Frames: map[string]data.Frame{ - "user_specified": frame, - }, - Errors: data.FrameErrors{Errors: errs}, - }, nil - } - // add the default - frames := make(map[string]data.Frame) - dirFrame, frameErrors := data.NewFileDirectoryFrame(DefaultLogsLocation, logPatterns) - frames["default"] = dirFrame - logFolders, errs := FindLogFileCandidates() - frameErrors = append(frameErrors, errs...) - i := 0 - for folder, paths := range logFolders { - // we will collect the default location anyway above so skip these - if folder != DefaultLogsLocation { - if collectArchives { - paths = append(paths, "*.gz") - } - dirFrame, errs := data.NewFileDirectoryFrame(folder, paths) - frames[fmt.Sprintf("logs-%d", i)] = dirFrame - frameErrors = append(frameErrors, errs...) - } - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: data.FrameErrors{Errors: frameErrors}, - }, err -} - -func (lc *LogsCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - config.BoolParam{ - Param: config.NewParam("collect_archives", "Collect compressed log archive files", false), - }, - }, - } -} - -func FindLogFileCandidates() (logFolders map[string][]string, configErrors []error) { - // we need the config to determine the location of the logs - configCandidates := make(map[string]data.ConfigFileFrame) - configFiles, err := FindConfigurationFiles() - logFolders = make(map[string][]string) - if err != nil { - configErrors = append(configErrors, err) - return logFolders, configErrors - } - for _, folder := range configFiles { - configFrame, errs := data.NewConfigFileFrame(folder) - configErrors = append(configErrors, errs...) - configCandidates[filepath.Clean(folder)] = configFrame - } - - for _, config := range configCandidates { - paths, errs := config.FindLogPaths() - for _, path := range paths { - folder := filepath.Dir(path) - filename := filepath.Base(path) - if _, ok := logFolders[folder]; !ok { - logFolders[folder] = []string{} - } - logFolders[folder] = utils.Unique(append(logFolders[folder], filename)) - } - configErrors = append(configErrors, errs...) - } - return logFolders, configErrors -} - -func (lc *LogsCollector) IsDefault() bool { - return true -} - -func (lc LogsCollector) Description() string { - return "Collects the ClickHouse logs directly from the database." -} - -// here we register the collector for use -func init() { - collectors.Register("logs", func() (collectors.Collector, error) { - return &LogsCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/logs_test.go deleted file mode 100644 index 5f0be734445..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/logs_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package clickhouse_test - -import ( - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestLogsConfiguration(t *testing.T) { - t.Run("correct configuration is returned for logs collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - logsCollector := clickhouse.NewLogsCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := logsCollector.Configuration() - require.Len(t, conf.Params, 2) - // check directory - require.IsType(t, config.StringParam{}, conf.Params[0]) - directory, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, directory.Required()) - require.Equal(t, directory.Name(), "directory") - require.Empty(t, directory.Value) - // check collect_archives - require.IsType(t, config.BoolParam{}, conf.Params[1]) - collectArchives, ok := conf.Params[1].(config.BoolParam) - require.True(t, ok) - require.False(t, collectArchives.Required()) - require.Equal(t, collectArchives.Name(), "collect_archives") - require.False(t, collectArchives.Value) - }) -} - -func TestLogsCollect(t *testing.T) { - - logsCollector := clickhouse.NewLogsCollector(&platform.ResourceManager{}) - - t.Run("test default logs collection", func(t *testing.T) { - // we can't rely on a local installation of clickhouse being present for tests - if it is present (and running) - // results maybe variable e.g. we may find a config. For now, we allow flexibility and test only default. - // TODO: we may want to test this within a container - bundle, err := logsCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, bundle) - // we will have some errors if clickhouse is installed or not. If former, permission issues - if latter missing folders. - require.Greater(t, len(bundle.Errors.Errors), 0) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "default") - _, ok := bundle.Frames["default"].(data.DirectoryFileFrame) - require.True(t, ok) - // no guarantees clickhouse is installed so this bundle could have no frames - }) - - t.Run("test logs collection when directory is specified", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - logsPath := path.Join(cwd, "../../../testdata", "logs", "var", "logs") - bundle, err := logsCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: logsPath, - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - }, - }) - require.Nil(t, err) - checkDirectoryBundle(t, bundle, logsPath, []string{"clickhouse-server.log", "clickhouse-server.err.log"}) - - }) - - t.Run("test logs collection of archives", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - logsPath := path.Join(cwd, "../../../testdata", "logs", "var", "logs") - bundle, err := logsCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: logsPath, - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("collect_archives", "Collect compressed log archive files", false), - }, - }, - }) - require.Nil(t, err) - checkDirectoryBundle(t, bundle, logsPath, []string{"clickhouse-server.log", "clickhouse-server.err.log", "clickhouse-server.log.gz"}) - }) - - t.Run("test when directory does not exist", func(t *testing.T) { - tmpDir := t.TempDir() - logsPath := path.Join(tmpDir, "random") - bundle, err := logsCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: logsPath, - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - }, - }) - // not a fatal error currently - require.Nil(t, err) - require.Len(t, bundle.Errors.Errors, 1) - require.Equal(t, fmt.Sprintf("directory %s does not exist", logsPath), bundle.Errors.Errors[0].Error()) - }) -} - -func checkDirectoryBundle(t *testing.T, bundle *data.DiagnosticBundle, logsPath string, expectedFiles []string) { - require.NotNil(t, bundle) - require.Nil(t, bundle.Errors.Errors) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "user_specified") - dirFrame, ok := bundle.Frames["user_specified"].(data.DirectoryFileFrame) - require.True(t, ok) - require.Equal(t, logsPath, dirFrame.Directory) - require.Equal(t, []string{"files"}, dirFrame.Columns()) - i := 0 - fullPaths := make([]string, len(expectedFiles)) - for i, filePath := range expectedFiles { - fullPaths[i] = path.Join(logsPath, filePath) - } - for { - values, ok, err := dirFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - file, ok := values[0].(data.SimpleFile) - require.True(t, ok) - require.Contains(t, fullPaths, file.FilePath()) - i += 1 - } - require.Equal(t, len(fullPaths), i) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/queries.json b/programs/diagnostics/internal/collectors/clickhouse/queries.json deleted file mode 100644 index f5cf4362c9e..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/queries.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "queries": { - "version": [ - { - "statement": "SELECT version()" - } - ], - "databases": [ - { - "statement": "SELECT name, engine, tables, partitions, parts, formatReadableSize(bytes_on_disk) \"disk_size\" FROM system.databases db LEFT JOIN ( SELECT database, uniq(table) \"tables\", uniq(table, partition) \"partitions\", count() AS parts, sum(bytes_on_disk) \"bytes_on_disk\" FROM system.parts WHERE active GROUP BY database ) AS db_stats ON db.name = db_stats.database ORDER BY bytes_on_disk DESC LIMIT {{.Limit}}" - } - ], - "access": [ - { - "statement": "SHOW ACCESS" - } - ], - "quotas": [ - { - "statement": "SHOW QUOTA" - } - ], - "db_engines": [ - { - "statement": "SELECT engine, count() \"count\" FROM system.databases GROUP BY engine" - } - ], - "table_engines": [ - { - "statement": "SELECT engine, count() \"count\" FROM system.tables WHERE database != 'system' GROUP BY engine" - } - ], - "dictionaries": [ - { - "statement": "SELECT source, type, status, count() \"count\" FROM system.dictionaries GROUP BY source, type, status ORDER BY status DESC, source" - } - ], - "replicated_tables_by_delay": [ - { - "statement": "SELECT database, table, is_leader, is_readonly, absolute_delay, queue_size, inserts_in_queue, merges_in_queue FROM system.replicas ORDER BY absolute_delay DESC LIMIT {{.Limit}}" - } - ], - "replication_queue_by_oldest": [ - { - "statement": "SELECT database, table, replica_name, position, node_name, type, source_replica, parts_to_merge, new_part_name, create_time, required_quorum, is_detach, is_currently_executing, num_tries, last_attempt_time, last_exception, concat( 'time: ', toString(last_postpone_time), ', number: ', toString(num_postponed), ', reason: ', postpone_reason ) postpone FROM system.replication_queue ORDER BY create_time ASC LIMIT {{.Limit}}" - } - ], - "replicated_fetches": [ - { - "statement": "SELECT database, table, round(elapsed, 1) \"elapsed\", round(100 * progress, 1) \"progress\", partition_id, result_part_name, result_part_path, total_size_bytes_compressed, bytes_read_compressed, source_replica_path, source_replica_hostname, source_replica_port, interserver_scheme, to_detached, thread_id FROM system.replicated_fetches" - } - ], - "tables_by_max_partition_count": [ - { - "statement": "SELECT database, table, count() \"partitions\", sum(part_count) \"parts\", max(part_count) \"max_parts_per_partition\" FROM ( SELECT database, table, partition, count() \"part_count\" FROM system.parts WHERE active GROUP BY database, table, partition ) partitions GROUP BY database, table ORDER BY max_parts_per_partition DESC LIMIT {{.Limit}}" - } - ], - "stack_traces": [ - { - "statement": "SELECT '\\n' || arrayStringConcat( arrayMap( x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace) ), '\\n' ) AS trace FROM system.stack_trace" - } - ], - "crash_log": [ - { - "statement": "SELECT event_time, signal, thread_id, query_id, '\\n' || arrayStringConcat(trace_full, '\\n') AS trace, version FROM system.crash_log ORDER BY event_time DESC" - } - ], - "merges": [ - { - "statement": "SELECT database, table, round(elapsed, 1) \"elapsed\", round(100 * progress, 1) \"progress\", is_mutation, partition_id, result_part_path, source_part_paths, num_parts, formatReadableSize(total_size_bytes_compressed) \"total_size_compressed\", formatReadableSize(bytes_read_uncompressed) \"read_uncompressed\", formatReadableSize(bytes_written_uncompressed) \"written_uncompressed\", columns_written, formatReadableSize(memory_usage) \"memory_usage\", thread_id FROM system.merges", - "constraint": ">=20.3" - }, - { - "statement": "SELECT database, table, round(elapsed, 1) \"elapsed\", round(100 * progress, 1) \"progress\", is_mutation, partition_id, num_parts, formatReadableSize(total_size_bytes_compressed) \"total_size_compressed\", formatReadableSize(bytes_read_uncompressed) \"read_uncompressed\", formatReadableSize(bytes_written_uncompressed) \"written_uncompressed\", columns_written, formatReadableSize(memory_usage) \"memory_usage\" FROM system.merges" - } - ], - "mutations": [ - { - "statement": "SELECT database, table, mutation_id, command, create_time, parts_to_do_names, parts_to_do, is_done, latest_failed_part, latest_fail_time, latest_fail_reason FROM system.mutations WHERE NOT is_done ORDER BY create_time DESC", - "constraint": ">=20.3" - }, - { - "statement": "SELECT database, table, mutation_id, command, create_time, parts_to_do, is_done, latest_failed_part, latest_fail_time, latest_fail_reason FROM system.mutations WHERE NOT is_done ORDER BY create_time DESC" - } - ], - "recent_data_parts": [ - { - "statement": "SELECT database, table, engine, partition_id, name, part_type, active, level, disk_name, path, marks, rows, bytes_on_disk, data_compressed_bytes, data_uncompressed_bytes, marks_bytes, modification_time, remove_time, refcount, is_frozen, min_date, max_date, min_time, max_time, min_block_number, max_block_number FROM system.parts WHERE modification_time > now() - INTERVAL 3 MINUTE ORDER BY modification_time DESC", - "constraint": ">=20.3" - }, - { - "statement": "SELECT database, table, engine, partition_id, name, active, level, path, marks, rows, bytes_on_disk, data_compressed_bytes, data_uncompressed_bytes, marks_bytes, modification_time, remove_time, refcount, is_frozen, min_date, max_date, min_time, max_time, min_block_number, max_block_number FROM system.parts WHERE modification_time > now() - INTERVAL 3 MINUTE ORDER BY modification_time DESC" - } - ], - "detached_parts": [ - { - "statement": "SELECT database, table, partition_id, name, disk, reason, min_block_number, max_block_number, level FROM system.detached_parts" - } - ], - "processes": [ - { - "statement": "SELECT elapsed, query_id, normalizeQuery(query) AS normalized_query, is_cancelled, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, formatReadableSize(memory_usage) AS \"memory usage\", user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, thread_ids, ProfileEvents, Settings FROM system.processes ORDER BY elapsed DESC", - "constraint": ">=21.8" - }, - { - "statement": "SELECT elapsed, query_id, normalizeQuery(query) AS normalized_query, is_cancelled, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, formatReadableSize(memory_usage) AS \"memory usage\", user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.processes ORDER BY elapsed DESC", - "constraint": ">=21.3" - }, - { - "statement": "SELECT elapsed, query_id, normalizeQuery(query) AS normalized_query, is_cancelled, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, formatReadableSize(memory_usage) AS \"memory usage\", user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.processes ORDER BY elapsed DESC" - } - ], - "top_queries_by_duration": [ - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents, Settings FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY query_duration_ms DESC LIMIT {{.Limit}}", - "constraint": ">=21.8" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY query_duration_ms DESC LIMIT {{.Limit}}", - "constraint": ">=21.3" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY query_duration_ms DESC LIMIT {{.Limit}}" - } - ], - "top_queries_by_memory": [ - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents, Settings FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY memory_usage DESC LIMIT {{.Limit}}", - "constraint": ">=21.8" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY memory_usage DESC LIMIT {{.Limit}}", - "constraint": ">=21.3" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY memory_usage DESC LIMIT {{.Limit}}" - } - ], - "failed_queries": [ - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents, Settings FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY AND exception != '' ORDER BY query_start_time DESC LIMIT {{.Limit}}", - "constraint": ">=21.8" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY AND exception != '' ORDER BY query_start_time DESC LIMIT {{.Limit}}", - "constraint": ">=21.3" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY AND exception != '' ORDER BY query_start_time DESC LIMIT {{.Limit}}" - } - ] - } -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/summary.go b/programs/diagnostics/internal/collectors/clickhouse/summary.go deleted file mode 100644 index 0b6dd3aff20..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/summary.go +++ /dev/null @@ -1,159 +0,0 @@ -package clickhouse - -import ( - "bytes" - _ "embed" - "encoding/json" - "strings" - "text/template" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/Masterminds/semver" - "github.com/pkg/errors" -) - -// This collector collects the system db from database - -type SummaryCollector struct { - resourceManager *platform.ResourceManager -} - -type querySet struct { - Queries map[string][]query `json:"queries"` -} - -type query struct { - Statement string `json:"statement"` - Constraint string `json:"constraint"` -} - -type ParameterTemplate struct { - Limit int64 -} - -//go:embed queries.json -var queryFile []byte - -func NewSummaryCollector(m *platform.ResourceManager) *SummaryCollector { - return &SummaryCollector{ - resourceManager: m, - } -} - -func (sc *SummaryCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(sc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - var queries querySet - err = json.Unmarshal(queryFile, &queries) - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrap(err, "Unable to read queries from disk") - } - limit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - paramTemplate := ParameterTemplate{ - Limit: limit, - } - frames := make(map[string]data.Frame) - - serverVersion, err := getServerSemVersion(sc) - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "Unable to read server version") - } - - var frameErrors []error - for queryId, sqlQueries := range queries.Queries { - // we find the first matching query that satisfies the current version. Empty version means ANY version is - // supported - for _, sqlQuery := range sqlQueries { - var queryConstraint *semver.Constraints - if sqlQuery.Constraint != "" { - queryConstraint, err = semver.NewConstraint(sqlQuery.Constraint) - if err != nil { - //we try another one - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to parse version %s for query %s", sqlQuery.Constraint, queryId)) - continue - } - } - if sqlQuery.Constraint == "" || queryConstraint.Check(serverVersion) { - tmpl, err := template.New(queryId).Parse(sqlQuery.Statement) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to parse query %s", queryId)) - //we try another one - continue - } - buf := new(bytes.Buffer) - err = tmpl.Execute(buf, paramTemplate) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to process query %s template", queryId)) - //we try another one - continue - } - frame, err := sc.resourceManager.DbClient.ExecuteStatement(queryId, buf.String()) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to execute query %s", queryId)) - //we try another one - } else { - frames[queryId] = frame - // only 1 query executed - break - } - } - } - - } - - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -func getServerSemVersion(sc *SummaryCollector) (*semver.Version, error) { - serverVersion, err := sc.resourceManager.DbClient.Version() - if err != nil { - return &semver.Version{}, err - } - //drop the build number - it is not a semantic version - versionComponents := strings.Split(serverVersion, ".") - serverVersion = strings.Join(versionComponents[:len(versionComponents)-1], ".") - return semver.NewVersion(serverVersion) -} - -func (sc *SummaryCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 20, - Param: config.NewParam("row_limit", "Limit rows on supported queries.", false), - }, - }, - } -} - -func (sc *SummaryCollector) IsDefault() bool { - return true -} - -func (sc *SummaryCollector) Description() string { - return "Collects summary statistics on the database based on a set of known useful queries." -} - -// here we register the collector for use -func init() { - collectors.Register("summary", func() (collectors.Collector, error) { - return &SummaryCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/summary_test.go b/programs/diagnostics/internal/collectors/clickhouse/summary_test.go deleted file mode 100644 index 92945d987ed..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/summary_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestSummaryConfiguration(t *testing.T) { - t.Run("correct configuration is returned for summary collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - summaryCollector := clickhouse.NewSummaryCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := summaryCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.IntParam{}, conf.Params[0]) - limit, ok := conf.Params[0].(config.IntParam) - require.True(t, ok) - require.False(t, limit.Required()) - require.Equal(t, limit.Name(), "row_limit") - require.Equal(t, int64(20), limit.Value) - }) -} - -func TestSummaryCollection(t *testing.T) { - - client := test.NewFakeClickhouseClient(make(map[string][]string)) - versionFrame := test.NewFakeDataFrame("version", []string{"version()"}, - [][]interface{}{ - {"22.1.3.7"}, - }, - ) - client.QueryResponses["SELECT version()"] = &versionFrame - databasesFrame := test.NewFakeDataFrame("databases", []string{"name", "engine", "tables", "partitions", "parts", "disk_size"}, - [][]interface{}{ - {"tutorial", "Atomic", 2, 2, 2, "1.70 GiB"}, - {"default", "Atomic", 5, 5, 6, "1.08 GiB"}, - {"system", "Atomic", 11, 24, 70, "1.05 GiB"}, - {"INFORMATION_SCHEMA", "Memory", 0, 0, 0, "0.00 B"}, - {"covid19db", "Atomic", 0, 0, 0, "0.00 B"}, - {"information_schema", "Memory", 0, 0, 0, "0.00 B"}}) - - client.QueryResponses["SELECT name, engine, tables, partitions, parts, formatReadableSize(bytes_on_disk) \"disk_size\" "+ - "FROM system.databases db LEFT JOIN ( SELECT database, uniq(table) \"tables\", uniq(table, partition) \"partitions\", "+ - "count() AS parts, sum(bytes_on_disk) \"bytes_on_disk\" FROM system.parts WHERE active GROUP BY database ) AS db_stats "+ - "ON db.name = db_stats.database ORDER BY bytes_on_disk DESC LIMIT 20"] = &databasesFrame - - summaryCollector := clickhouse.NewSummaryCollector(&platform.ResourceManager{ - DbClient: client, - }) - - t.Run("test default summary collection", func(t *testing.T) { - bundle, errs := summaryCollector.Collect(config.Configuration{}) - require.Empty(t, errs) - require.Len(t, bundle.Errors.Errors, 30) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 2) - // check version frame - require.Contains(t, bundle.Frames, "version") - require.Equal(t, []string{"version()"}, bundle.Frames["version"].Columns()) - checkFrame(t, bundle.Frames["version"], versionFrame.Rows) - //check databases frame - require.Contains(t, bundle.Frames, "databases") - require.Equal(t, []string{"name", "engine", "tables", "partitions", "parts", "disk_size"}, bundle.Frames["databases"].Columns()) - checkFrame(t, bundle.Frames["databases"], databasesFrame.Rows) - client.Reset() - }) - - t.Run("test summary collection with limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 1, - Param: config.NewParam("row_limit", "Limit rows on supported queries.", false), - }, - }, - } - bundle, errs := summaryCollector.Collect(conf) - - require.Empty(t, errs) - require.Len(t, bundle.Errors.Errors, 31) - require.NotNil(t, bundle) - // databases will be absent due to limit - require.Len(t, bundle.Frames, 1) - // check version frame - require.Contains(t, bundle.Frames, "version") - require.Equal(t, []string{"version()"}, bundle.Frames["version"].Columns()) - checkFrame(t, bundle.Frames["version"], versionFrame.Rows) - - client.QueryResponses["SELECT name, engine, tables, partitions, parts, formatReadableSize(bytes_on_disk) \"disk_size\" "+ - "FROM system.databases db LEFT JOIN ( SELECT database, uniq(table) \"tables\", uniq(table, partition) \"partitions\", "+ - "count() AS parts, sum(bytes_on_disk) \"bytes_on_disk\" FROM system.parts WHERE active GROUP BY database ) AS db_stats "+ - "ON db.name = db_stats.database ORDER BY bytes_on_disk DESC LIMIT 1"] = &databasesFrame - bundle, errs = summaryCollector.Collect(conf) - require.Empty(t, errs) - require.Len(t, bundle.Errors.Errors, 30) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 2) - require.Contains(t, bundle.Frames, "version") - //check databases frame - require.Contains(t, bundle.Frames, "databases") - require.Equal(t, []string{"name", "engine", "tables", "partitions", "parts", "disk_size"}, bundle.Frames["databases"].Columns()) - // this will parse as our mock client does not read statement (specifically the limit clause) when called with execute - checkFrame(t, bundle.Frames["databases"], databasesFrame.Rows) - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/system.go b/programs/diagnostics/internal/collectors/clickhouse/system.go deleted file mode 100644 index d47cfd924f3..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/system.go +++ /dev/null @@ -1,165 +0,0 @@ -package clickhouse - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" -) - -// This collector collects the system db from database - -type SystemDatabaseCollector struct { - resourceManager *platform.ResourceManager -} - -const SystemDatabase = "system" - -// ExcludeColumns columns if we need - this will be refined over time [table_name][columnA, columnB] -var ExcludeColumns = map[string][]string{} - -// BannedTables - Hardcoded list. These are always excluded even if the user doesn't specify in exclude_tables. -//Attempts to export will work but we will warn -var BannedTables = []string{"numbers", "zeros"} - -// OrderBy contains a map of tables to an order by clause - by default we don't order table dumps -var OrderBy = map[string]data.OrderBy{ - "errors": { - Column: "last_error_message", - Order: data.Desc, - }, - "replication_queue": { - Column: "create_time", - Order: data.Asc, - }, -} - -func NewSystemDatabaseCollector(m *platform.ResourceManager) *SystemDatabaseCollector { - return &SystemDatabaseCollector{ - resourceManager: m, - } -} - -func (sc *SystemDatabaseCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(sc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - includeTables, err := config.ReadStringListValues(conf, "include_tables") - if err != nil { - return &data.DiagnosticBundle{}, err - } - excludeTables, err := config.ReadStringListValues(conf, "exclude_tables") - if err != nil { - return &data.DiagnosticBundle{}, err - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - excludeTables = checkBannedTables(includeTables, excludeTables) - ds, err := sc.readSystemAllTables(includeTables, excludeTables, rowLimit) - if err != nil { - return &data.DiagnosticBundle{}, err - } - return ds, nil -} - -// all banned tables are added to excluded if not present and not specified in included. Returns new exclude_tables list. -func checkBannedTables(includeTables []string, excludeTables []string) []string { - for _, bannedTable := range BannedTables { - //if its specified we don't add to our exclude list - explicitly included tables take precedence - if !utils.Contains(includeTables, bannedTable) && !utils.Contains(excludeTables, bannedTable) { - excludeTables = append(excludeTables, bannedTable) - } - } - return excludeTables -} - -func (sc *SystemDatabaseCollector) readSystemAllTables(include []string, exclude []string, limit int64) (*data.DiagnosticBundle, error) { - tableNames, err := sc.resourceManager.DbClient.ReadTableNamesForDatabase(SystemDatabase) - if err != nil { - return nil, err - } - var frameErrors []error - if include != nil { - // nil means include everything - tableNames = utils.Intersection(tableNames, include) - if len(tableNames) != len(include) { - // we warn that some included tables aren't present in db - frameErrors = append(frameErrors, fmt.Errorf("some tables specified in the include_tables are not in the system database and will not be exported: %v", - utils.Distinct(include, tableNames))) - } - } - - // exclude tables unless specified in includes - excludedTables := utils.Distinct(exclude, include) - tableNames = utils.Distinct(tableNames, excludedTables) - frames := make(map[string]data.Frame) - - for _, tableName := range tableNames { - var excludeColumns []string - if _, ok := ExcludeColumns[tableName]; ok { - excludeColumns = ExcludeColumns[tableName] - } - orderBy := data.OrderBy{} - if _, ok := OrderBy[tableName]; ok { - orderBy = OrderBy[tableName] - } - frame, err := sc.resourceManager.DbClient.ReadTable(SystemDatabase, tableName, excludeColumns, orderBy, limit) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to collect %s", tableName)) - } else { - frames[tableName] = frame - } - } - - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -func (sc *SystemDatabaseCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect. Takes precedence over exclude_tables. If not specified (default) all tables except exclude_tables.", false), - }, - config.StringListParam{ - Values: []string{"licenses", "distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper", "aggregate_function_combinators", "collations", "contributors", "data_type_families", "formats", "graphite_retentions", "numbers", "numbers_mt", "one", "parts_columns", "projection_parts", "projection_parts_columns", "table_engines", "time_zones", "zeros", "zeros_mt"}, - Param: config.NewParam("exclude_tables", "Specify list of tables to not collect.", false), - }, - config.IntParam{ - Value: 100000, - Param: config.NewParam("row_limit", "Maximum number of rows to collect from any table. Negative values mean unlimited.", false), - }, - }, - } -} - -func (sc *SystemDatabaseCollector) IsDefault() bool { - return true -} - -func (sc *SystemDatabaseCollector) Description() string { - return "Collects all tables in the system database, except those which have been excluded." -} - -// here we register the collector for use -func init() { - collectors.Register("system_db", func() (collectors.Collector, error) { - return &SystemDatabaseCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/system_test.go b/programs/diagnostics/internal/collectors/clickhouse/system_test.go deleted file mode 100644 index d1b9a6e7859..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/system_test.go +++ /dev/null @@ -1,366 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestSystemConfiguration(t *testing.T) { - t.Run("correct configuration is returned for system db collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - systemDbCollector := clickhouse.NewSystemDatabaseCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := systemDbCollector.Configuration() - require.Len(t, conf.Params, 3) - // check first param - require.IsType(t, config.StringListParam{}, conf.Params[0]) - includeTables, ok := conf.Params[0].(config.StringListParam) - require.True(t, ok) - require.False(t, includeTables.Required()) - require.Equal(t, includeTables.Name(), "include_tables") - require.Nil(t, includeTables.Values) - // check second param - require.IsType(t, config.StringListParam{}, conf.Params[1]) - excludeTables, ok := conf.Params[1].(config.StringListParam) - require.True(t, ok) - require.False(t, excludeTables.Required()) - require.Equal(t, "exclude_tables", excludeTables.Name()) - require.Equal(t, []string{"licenses", "distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper", "aggregate_function_combinators", "collations", "contributors", "data_type_families", "formats", "graphite_retentions", "numbers", "numbers_mt", "one", "parts_columns", "projection_parts", "projection_parts_columns", "table_engines", "time_zones", "zeros", "zeros_mt"}, excludeTables.Values) - // check third param - require.IsType(t, config.IntParam{}, conf.Params[2]) - rowLimit, ok := conf.Params[2].(config.IntParam) - require.True(t, ok) - require.False(t, rowLimit.Required()) - require.Equal(t, "row_limit", rowLimit.Name()) - require.Equal(t, int64(100000), rowLimit.Value) - }) -} - -func TestSystemDbCollect(t *testing.T) { - - diskFrame := test.NewFakeDataFrame("disks", []string{"name", "path", "free_space", "total_space", "keep_free_space", "type"}, - [][]interface{}{ - {"default", "/var/lib/clickhouse", 1729659346944, 1938213220352, "", "local"}, - }, - ) - clusterFrame := test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - {"events", 2, 1, 1, "dalem-local-clickhouse-blue-2", "192.168.144.4", 9000, 1, "default", "", 0, 0, 0}, - {"events", 3, 1, 1, "dalem-local-clickhouse-blue-3", "192.168.144.3", 9000, 1, "default", "", 0, 0, 0}, - }, - ) - userFrame := test.NewFakeDataFrame("users", []string{"name", "id", "storage", "auth_type", "auth_params", "host_ip", "host_names", "host_names_regexp", "host_names_like"}, - [][]interface{}{ - {"default", "94309d50-4f52-5250-31bd-74fecac179db,users.xml,plaintext_password", "sha256_password", []string{"::0"}, []string{}, []string{}, []string{}}, - }, - ) - - dbTables := map[string][]string{ - clickhouse.SystemDatabase: {"disks", "clusters", "users"}, - } - client := test.NewFakeClickhouseClient(dbTables) - - client.QueryResponses["SELECT * FROM system.disks LIMIT 100000"] = &diskFrame - client.QueryResponses["SELECT * FROM system.clusters LIMIT 100000"] = &clusterFrame - client.QueryResponses["SELECT * FROM system.users LIMIT 100000"] = &userFrame - systemDbCollector := clickhouse.NewSystemDatabaseCollector(&platform.ResourceManager{ - DbClient: client, - }) - - t.Run("test default system db collection", func(t *testing.T) { - diagSet, err := systemDbCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - checkFrame(t, diagSet.Frames["clusters"], clusterFrame.Rows) - // users frame - require.Equal(t, "users", diagSet.Frames["users"].Name()) - require.Equal(t, userFrame.ColumnNames, diagSet.Frames["users"].Columns()) - checkFrame(t, diagSet.Frames["users"], userFrame.Rows) - client.Reset() - }) - - t.Run("test when we pass an includes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks"}, - Param: config.NewParam("include_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 1) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - client.Reset() - }) - - // test excludes - t.Run("test when we pass an excludes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - Values: []string{"disks"}, - Param: config.NewParam("exclude_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 2) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - checkFrame(t, diagSet.Frames["clusters"], clusterFrame.Rows) - // users frame - require.Equal(t, "users", diagSet.Frames["users"].Name()) - require.Equal(t, userFrame.ColumnNames, diagSet.Frames["users"].Columns()) - checkFrame(t, diagSet.Frames["users"], userFrame.Rows) - client.Reset() - }) - - // test includes which isn't in the list - t.Run("test when we pass an invalid includes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks", "invalid"}, - Param: config.NewParam("include_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 1) - require.Equal(t, diagSet.Errors.Error(), "some tables specified in the include_tables are not in the "+ - "system database and will not be exported: [invalid]") - require.Len(t, diagSet.Frames, 1) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - client.Reset() - }) - - t.Run("test when we use a table with excluded fields", func(t *testing.T) { - excludeDefault := clickhouse.ExcludeColumns - client.QueryResponses["SELECT * EXCEPT(keep_free_space,type) FROM system.disks LIMIT 100000"] = &diskFrame - clickhouse.ExcludeColumns = map[string][]string{ - "disks": {"keep_free_space", "type"}, - } - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks"}, - Param: config.NewParam("include_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 1) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, []string{"name", "path", "free_space", "total_space"}, diagSet.Frames["disks"].Columns()) - eDiskFrame := test.NewFakeDataFrame("disks", []string{"name", "path", "free_space", "total_space"}, - [][]interface{}{ - {"default", "/var/lib/clickhouse", 1729659346944, 1938213220352}, - }, - ) - checkFrame(t, diagSet.Frames["disks"], eDiskFrame.Rows) - clickhouse.ExcludeColumns = excludeDefault - client.Reset() - }) - - t.Run("test with a low row limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 1, - Param: config.NewParam("row_limit", "Maximum number of rows to collect from any table. Negative values mean unlimited.", false), - }, - }, - } - client.QueryResponses["SELECT * FROM system.disks LIMIT 1"] = &diskFrame - client.QueryResponses["SELECT * FROM system.clusters LIMIT 1"] = &clusterFrame - client.QueryResponses["SELECT * FROM system.users LIMIT 1"] = &userFrame - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - lClusterFrame := test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - }) - checkFrame(t, diagSet.Frames["clusters"], lClusterFrame.Rows) - client.Reset() - }) - - t.Run("test with a negative low row limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: -23, - Param: config.NewParam("row_limit", "Maximum number of rows to collect from any table. Negative values mean unlimited.", false), - }, - }, - } - client.QueryResponses["SELECT * FROM system.clusters"] = &clusterFrame - client.QueryResponses["SELECT * FROM system.disks"] = &diskFrame - client.QueryResponses["SELECT * FROM system.users"] = &userFrame - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - checkFrame(t, diagSet.Frames["clusters"], clusterFrame.Rows) - // users frame - require.Equal(t, "users", diagSet.Frames["users"].Name()) - require.Equal(t, userFrame.ColumnNames, diagSet.Frames["users"].Columns()) - checkFrame(t, diagSet.Frames["users"], userFrame.Rows) - client.Reset() - }) - - t.Run("test that includes overrides excludes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks"}, - Param: config.NewParam("exclude_tables", "Excluded", false), - }, - config.StringListParam{ - // nil means include everything - Values: []string{"disks", "clusters", "users"}, - Param: config.NewParam("include_tables", "Included", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - client.Reset() - }) - - t.Run("test banned", func(t *testing.T) { - bannedDefault := clickhouse.BannedTables - clickhouse.BannedTables = []string{"disks"} - diagSet, err := systemDbCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 2) - require.Contains(t, diagSet.Frames, "users") - require.Contains(t, diagSet.Frames, "clusters") - clickhouse.BannedTables = bannedDefault - client.Reset() - }) - - t.Run("test banned unless included", func(t *testing.T) { - bannedDefault := clickhouse.BannedTables - clickhouse.BannedTables = []string{"disks"} - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks", "clusters", "users"}, - Param: config.NewParam("include_tables", "Included", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - require.Contains(t, diagSet.Frames, "disks") - require.Contains(t, diagSet.Frames, "users") - require.Contains(t, diagSet.Frames, "clusters") - clickhouse.BannedTables = bannedDefault - client.Reset() - }) - - t.Run("tables are ordered if configured", func(t *testing.T) { - defaultOrderBy := clickhouse.OrderBy - clickhouse.OrderBy = map[string]data.OrderBy{ - "clusters": { - Column: "shard_num", - Order: data.Desc, - }, - } - client.QueryResponses["SELECT * FROM system.clusters ORDER BY shard_num DESC LIMIT 100000"] = &clusterFrame - diagSet, err := systemDbCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - clickhouse.OrderBy = defaultOrderBy - oClusterFrame := test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 3, 1, 1, "dalem-local-clickhouse-blue-3", "192.168.144.3", 9000, 1, "default", "", 0, 0, 0}, - {"events", 2, 1, 1, "dalem-local-clickhouse-blue-2", "192.168.144.4", 9000, 1, "default", "", 0, 0, 0}, - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - }, - ) - checkFrame(t, diagSet.Frames["clusters"], oClusterFrame.Rows) - client.Reset() - }) - -} - -func checkFrame(t *testing.T, frame data.Frame, rows [][]interface{}) { - i := 0 - for { - values, ok, err := frame.Next() - require.Nil(t, err) - if !ok { - break - } - require.ElementsMatch(t, rows[i], values) - i += 1 - } - require.Equal(t, i, len(rows)) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go deleted file mode 100644 index 78aefeaa0c1..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go +++ /dev/null @@ -1,153 +0,0 @@ -package clickhouse - -import ( - "fmt" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/bmatcuk/doublestar/v4" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -// This collector collects the system zookeeper db - -type ZookeeperCollector struct { - resourceManager *platform.ResourceManager -} - -func NewZookeeperCollector(m *platform.ResourceManager) *ZookeeperCollector { - return &ZookeeperCollector{ - resourceManager: m, - } -} - -func (zkc *ZookeeperCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(zkc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - - pathPattern, err := config.ReadStringValue(conf, "path_pattern") - if err != nil { - return &data.DiagnosticBundle{}, err - } - defaultPattern, _ := zkc.Configuration().GetConfigParam("path_pattern") - if defaultPattern.(config.StringParam).Value != pathPattern { - log.Warn().Msgf("Using non default zookeeper glob pattern [%s] - this can potentially cause high query load", pathPattern) - } - maxDepth, err := config.ReadIntValue(conf, "max_depth") - if err != nil { - return &data.DiagnosticBundle{}, err - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - // we use doublestar for globs as it provides us with ** but also allows us to identify prefix or base paths - if !doublestar.ValidatePattern(pathPattern) { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "%s is not a valid pattern", pathPattern) - } - base, _ := doublestar.SplitPattern(pathPattern) - frames := make(map[string]data.Frame) - hFrame, frameErrors := zkc.collectSubFrames(base, pathPattern, rowLimit, 0, maxDepth) - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - frames["zookeeper_db"] = hFrame - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -// recursively iterates over the zookeeper sub tables to a max depth, applying the filter and max rows per table -func (zkc *ZookeeperCollector) collectSubFrames(path, pathPattern string, rowLimit, currentDepth, maxDepth int64) (data.HierarchicalFrame, []error) { - var frameErrors []error - var subFrames []data.HierarchicalFrame - - currentDepth += 1 - if currentDepth == maxDepth { - return data.HierarchicalFrame{}, frameErrors - } - match, err := doublestar.PathMatch(pathPattern, path) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Path match failed for pattern %s with path %s", pathPattern, path)) - return data.HierarchicalFrame{}, frameErrors - } - // we allow a single level to be examined or we never get going - if !match && currentDepth > 1 { - return data.HierarchicalFrame{}, frameErrors - } - frame, err := zkc.resourceManager.DbClient.ExecuteStatement(path, fmt.Sprintf("SELECT name FROM system.zookeeper WHERE path='%s' LIMIT %d", path, rowLimit)) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to read zookeeper table path for sub paths %s", path)) - return data.HierarchicalFrame{}, frameErrors - } - - // this isn't ideal, we add re-execute the query to our collection as this will be consumed by the output lazily - outputFrame, err := zkc.resourceManager.DbClient.ExecuteStatement(path, fmt.Sprintf("SELECT * FROM system.zookeeper WHERE path='%s' LIMIT %d", path, rowLimit)) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to read zookeeper table path %s", path)) - return data.HierarchicalFrame{}, frameErrors - } - frameComponents := strings.Split(path, "/") - frameId := frameComponents[len(frameComponents)-1] - - for { - values, ok, err := frame.Next() - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "unable to read frame %s", frame.Name())) - return data.NewHierarchicalFrame(frameId, outputFrame, subFrames), frameErrors - } - if !ok { - return data.NewHierarchicalFrame(frameId, outputFrame, subFrames), frameErrors - } - subName := fmt.Sprintf("%v", values[0]) - subPath := fmt.Sprintf("%s/%s", path, subName) - subFrame, errs := zkc.collectSubFrames(subPath, pathPattern, rowLimit, currentDepth, maxDepth) - if subFrame.Name() != "" { - subFrames = append(subFrames, subFrame) - } - frameErrors = append(frameErrors, errs...) - } -} - -func (zkc *ZookeeperCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "/clickhouse/{task_queue}/**", - Param: config.NewParam("path_pattern", "Glob pattern for zookeeper path matching. Change with caution.", false), - }, - config.IntParam{ - Value: 8, - Param: config.NewParam("max_depth", "Max depth for zookeeper navigation.", false), - }, - config.IntParam{ - Value: 10, - Param: config.NewParam("row_limit", "Maximum number of rows/sub nodes to collect/expand from any zookeeper leaf. Negative values mean unlimited.", false), - }, - }, - } -} - -func (zkc *ZookeeperCollector) IsDefault() bool { - return false -} - -func (zkc *ZookeeperCollector) Description() string { - return "Collects Zookeeper information available within ClickHouse." -} - -// here we register the collector for use -func init() { - collectors.Register("zookeeper_db", func() (collectors.Collector, error) { - return &ZookeeperCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go deleted file mode 100644 index 3e56f6200f0..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestZookeeperConfiguration(t *testing.T) { - t.Run("correct configuration is returned for system zookeeper collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - zkCollector := clickhouse.NewZookeeperCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := zkCollector.Configuration() - require.Len(t, conf.Params, 3) - // check first param - require.IsType(t, config.StringParam{}, conf.Params[0]) - pathPattern, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, pathPattern.Required()) - require.Equal(t, pathPattern.Name(), "path_pattern") - require.Equal(t, "/clickhouse/{task_queue}/**", pathPattern.Value) - // check second param - require.IsType(t, config.IntParam{}, conf.Params[1]) - maxDepth, ok := conf.Params[1].(config.IntParam) - require.True(t, ok) - require.False(t, maxDepth.Required()) - require.Equal(t, "max_depth", maxDepth.Name()) - require.Equal(t, int64(8), maxDepth.Value) - // check third param - require.IsType(t, config.IntParam{}, conf.Params[2]) - rowLimit, ok := conf.Params[2].(config.IntParam) - require.True(t, ok) - require.False(t, rowLimit.Required()) - require.Equal(t, "row_limit", rowLimit.Name()) - require.Equal(t, int64(10), rowLimit.Value) - }) -} - -func TestZookeeperCollect(t *testing.T) { - level1 := test.NewFakeDataFrame("level_1", []string{"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - [][]interface{}{ - {"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - {"task_queue", "", "4", "4", "2022-02-22 13:30:15", "2022-02-22 13:30:15", "0", "1", "0", "0", "0", "1", "5", "/clickhouse"}, - {"copytasks", "", "525608", "525608", "2022-03-09 13:47:39", "2022-03-09 13:47:39", "0", "7", "0", "0", "0", "7", "526100", "/clickhouse"}, - }, - ) - level2 := test.NewFakeDataFrame("level_2", []string{"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - [][]interface{}{ - {"ddl", "", "5", "5", "2022-02-22 13:30:15", "2022-02-22 13:30:15", "0", "0", "0", "0", "0", "0", "5", "/clickhouse/task_queue"}, - }, - ) - level3 := test.NewFakeDataFrame("level_2", []string{"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - [][]interface{}{}, - ) - dbTables := map[string][]string{ - clickhouse.SystemDatabase: {"zookeeper"}, - } - client := test.NewFakeClickhouseClient(dbTables) - - client.QueryResponses["SELECT name FROM system.zookeeper WHERE path='/clickhouse' LIMIT 10"] = &level1 - // can't reuse the frame as the first frame will be iterated as part of the recursive zookeeper search performed by the collector - cLevel1 := test.NewFakeDataFrame("level_1", level1.Columns(), level1.Rows) - client.QueryResponses["SELECT * FROM system.zookeeper WHERE path='/clickhouse' LIMIT 10"] = &cLevel1 - client.QueryResponses["SELECT name FROM system.zookeeper WHERE path='/clickhouse/task_queue' LIMIT 10"] = &level2 - cLevel2 := test.NewFakeDataFrame("level_2", level2.Columns(), level2.Rows) - client.QueryResponses["SELECT * FROM system.zookeeper WHERE path='/clickhouse/task_queue' LIMIT 10"] = &cLevel2 - client.QueryResponses["SELECT name FROM system.zookeeper WHERE path='/clickhouse/task_queue/ddl' LIMIT 10"] = &level3 - cLevel3 := test.NewFakeDataFrame("level_3", level3.Columns(), level3.Rows) - client.QueryResponses["SELECT * FROM system.zookeeper WHERE path='/clickhouse/task_queue/ddl' LIMIT 10"] = &cLevel3 - - zKCollector := clickhouse.NewZookeeperCollector(&platform.ResourceManager{ - DbClient: client, - }) - - t.Run("test default zookeeper collection", func(t *testing.T) { - diagSet, err := zKCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 1) - require.Contains(t, diagSet.Frames, "zookeeper_db") - require.Equal(t, "clickhouse", diagSet.Frames["zookeeper_db"].Name()) - require.IsType(t, data.HierarchicalFrame{}, diagSet.Frames["zookeeper_db"]) - checkFrame(t, diagSet.Frames["zookeeper_db"], level1.Rows) - require.Equal(t, level1.Columns(), diagSet.Frames["zookeeper_db"].Columns()) - hierarchicalFrame := diagSet.Frames["zookeeper_db"].(data.HierarchicalFrame) - require.Len(t, hierarchicalFrame.SubFrames, 1) - checkFrame(t, hierarchicalFrame.SubFrames[0], cLevel2.Rows) - require.Equal(t, cLevel2.Columns(), hierarchicalFrame.SubFrames[0].Columns()) - hierarchicalFrame = hierarchicalFrame.SubFrames[0] - require.Len(t, hierarchicalFrame.SubFrames, 1) - checkFrame(t, hierarchicalFrame.SubFrames[0], cLevel3.Rows) - require.Equal(t, cLevel3.Columns(), hierarchicalFrame.SubFrames[0].Columns()) - }) -} diff --git a/programs/diagnostics/internal/collectors/registry.go b/programs/diagnostics/internal/collectors/registry.go deleted file mode 100644 index 5611f947466..00000000000 --- a/programs/diagnostics/internal/collectors/registry.go +++ /dev/null @@ -1,75 +0,0 @@ -package collectors - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -type Collector interface { - Collect(config config.Configuration) (*data.DiagnosticBundle, error) - Configuration() config.Configuration - IsDefault() bool - Description() string -} - -// Register can be called from init() on a collector in this package -// It will automatically be added to the Collectors map to be called externally -func Register(name string, collector CollectorFactory) { - if name == "diag_trace" { - // we use this to record errors and warnings - log.Fatal().Msgf("diag_trace is a reserved collector name") - } - // names must be unique - if _, ok := Collectors[name]; ok { - log.Fatal().Msgf("More than 1 collector is trying to register under the name %s. Names must be unique.", name) - } - Collectors[name] = collector -} - -// CollectorFactory lets us use a closure to get instances of the collector struct -type CollectorFactory func() (Collector, error) - -var Collectors = map[string]CollectorFactory{} - -func GetCollectorNames(defaultOnly bool) []string { - // can't pre-allocate as not all maybe default - var collectors []string - for collectorName := range Collectors { - collector, err := GetCollectorByName(collectorName) - if err != nil { - log.Fatal().Err(err) - } - if !defaultOnly || (defaultOnly && collector.IsDefault()) { - collectors = append(collectors, collectorName) - } - } - return collectors -} - -func GetCollectorByName(name string) (Collector, error) { - if collectorFactory, ok := Collectors[name]; ok { - //do something here - collector, err := collectorFactory() - if err != nil { - return nil, errors.Wrapf(err, "collector %s could not be initialized", name) - } - return collector, nil - } - return nil, fmt.Errorf("%s is not a valid collector name", name) -} - -func BuildConfigurationOptions() (map[string]config.Configuration, error) { - configurations := make(map[string]config.Configuration) - for name, collectorFactory := range Collectors { - collector, err := collectorFactory() - if err != nil { - return nil, errors.Wrapf(err, "collector %s could not be initialized", name) - } - configurations[name] = collector.Configuration() - } - return configurations, nil -} diff --git a/programs/diagnostics/internal/collectors/registry_test.go b/programs/diagnostics/internal/collectors/registry_test.go deleted file mode 100644 index eccc5f2265d..00000000000 --- a/programs/diagnostics/internal/collectors/registry_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package collectors_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/stretchr/testify/require" -) - -func TestGetCollectorNames(t *testing.T) { - t.Run("can get all collector names", func(t *testing.T) { - collectorNames := collectors.GetCollectorNames(false) - require.ElementsMatch(t, []string{"system_db", "config", "summary", "system", "logs", "db_logs", "file", "command", "zookeeper_db"}, collectorNames) - }) - - t.Run("can get default collector names", func(t *testing.T) { - collectorNames := collectors.GetCollectorNames(true) - require.ElementsMatch(t, []string{"system_db", "config", "summary", "system", "logs", "db_logs"}, collectorNames) - }) -} - -func TestGetCollectorByName(t *testing.T) { - - t.Run("can get collector by name", func(t *testing.T) { - collector, err := collectors.GetCollectorByName("system_db") - require.Nil(t, err) - require.Equal(t, clickhouse.NewSystemDatabaseCollector(platform.GetResourceManager()), collector) - }) - - t.Run("fails on non existing collector", func(t *testing.T) { - collector, err := collectors.GetCollectorByName("random") - require.NotNil(t, err) - require.Equal(t, "random is not a valid collector name", err.Error()) - require.Nil(t, collector) - }) -} - -func TestBuildConfigurationOptions(t *testing.T) { - - t.Run("can get all collector configurations", func(t *testing.T) { - configs, err := collectors.BuildConfigurationOptions() - require.Nil(t, err) - require.Len(t, configs, 9) - require.Contains(t, configs, "system_db") - require.Contains(t, configs, "config") - require.Contains(t, configs, "summary") - require.Contains(t, configs, "system") - require.Contains(t, configs, "logs") - require.Contains(t, configs, "db_logs") - require.Contains(t, configs, "file") - require.Contains(t, configs, "command") - require.Contains(t, configs, "zookeeper_db") - }) -} diff --git a/programs/diagnostics/internal/collectors/system/command.go b/programs/diagnostics/internal/collectors/system/command.go deleted file mode 100644 index ba4dd1e996c..00000000000 --- a/programs/diagnostics/internal/collectors/system/command.go +++ /dev/null @@ -1,90 +0,0 @@ -package system - -import ( - "bytes" - "os/exec" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/google/shlex" - "github.com/pkg/errors" -) - -// This collector runs a user specified command and collects it to a file - -type CommandCollector struct { - resourceManager *platform.ResourceManager -} - -func NewCommandCollector(m *platform.ResourceManager) *CommandCollector { - return &CommandCollector{ - resourceManager: m, - } -} - -func (c *CommandCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(c.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - command, err := config.ReadStringValue(conf, "command") - if err != nil { - return &data.DiagnosticBundle{}, err - } - var frameErrors []error - // shlex to split the commands and args - cmdArgs, err := shlex.Split(command) - if err != nil || len(cmdArgs) == 0 { - return &data.DiagnosticBundle{}, errors.Wrap(err, "Unable to parse command") - } - cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err = cmd.Run() - var sError string - if err != nil { - frameErrors = append(frameErrors, errors.Wrap(err, "Unable to execute command")) - sError = err.Error() - } - memoryFrame := data.NewMemoryFrame("output", []string{"command", "stdout", "stderr", "error"}, [][]interface{}{ - {command, stdout.String(), stderr.String(), sError}, - }) - return &data.DiagnosticBundle{ - Errors: data.FrameErrors{Errors: frameErrors}, - Frames: map[string]data.Frame{ - "output": memoryFrame, - }, - }, nil -} - -func (c *CommandCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - } -} - -func (c *CommandCollector) IsDefault() bool { - return false -} - -func (c *CommandCollector) Description() string { - return "Allows collection of the output from a user specified command" -} - -// here we register the collector for use -func init() { - collectors.Register("command", func() (collectors.Collector, error) { - return &CommandCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/system/command_test.go b/programs/diagnostics/internal/collectors/system/command_test.go deleted file mode 100644 index 7de00cdabf4..00000000000 --- a/programs/diagnostics/internal/collectors/system/command_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package system_test - -import ( - "fmt" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestCommandConfiguration(t *testing.T) { - t.Run("correct configuration is returned for file collector", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - conf := commandCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.StringParam{}, conf.Params[0]) - command, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.True(t, command.Required()) - require.Equal(t, command.Name(), "command") - require.Equal(t, "", command.Value) - }) -} - -func TestCommandCollect(t *testing.T) { - t.Run("test simple command with args", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - bundle, err := commandCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "ls -l ../../../testdata", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - }) - require.Nil(t, err) - require.Nil(t, bundle.Errors.Errors) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "output") - require.Equal(t, bundle.Frames["output"].Columns(), []string{"command", "stdout", "stderr", "error"}) - memFrame := bundle.Frames["output"].(data.MemoryFrame) - values, ok, err := memFrame.Next() - require.True(t, ok) - require.Nil(t, err) - fmt.Println(values) - require.Len(t, values, 4) - require.Equal(t, "ls -l ../../../testdata", values[0]) - require.Contains(t, values[1], "configs") - require.Contains(t, values[1], "docker") - require.Contains(t, values[1], "log") - require.Equal(t, "", values[2]) - require.Equal(t, "", values[3]) - values, ok, err = memFrame.Next() - require.False(t, ok) - require.Nil(t, err) - require.Nil(t, values) - }) - - t.Run("test empty command", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - bundle, err := commandCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - }) - require.Equal(t, "parameter command is invalid - command cannot be empty", err.Error()) - require.Equal(t, &data.DiagnosticBundle{}, bundle) - }) - - t.Run("test invalid command", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - bundle, err := commandCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "ls --invalid ../../../testdata", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - }) - // commands may error with output - we still capture on stderr - require.Nil(t, err) - require.Len(t, bundle.Errors.Errors, 1) - require.Equal(t, "Unable to execute command: exit status 2", bundle.Errors.Errors[0].Error()) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "output") - require.Equal(t, bundle.Frames["output"].Columns(), []string{"command", "stdout", "stderr", "error"}) - memFrame := bundle.Frames["output"].(data.MemoryFrame) - values, ok, err := memFrame.Next() - require.True(t, ok) - require.Nil(t, err) - require.Len(t, values, 4) - require.Equal(t, "ls --invalid ../../../testdata", values[0]) - require.Equal(t, "", values[1]) - // exact values here may vary on platform - require.NotEmpty(t, values[2]) - require.NotEmpty(t, values[3]) - }) -} diff --git a/programs/diagnostics/internal/collectors/system/file.go b/programs/diagnostics/internal/collectors/system/file.go deleted file mode 100644 index cda91636c52..00000000000 --- a/programs/diagnostics/internal/collectors/system/file.go +++ /dev/null @@ -1,100 +0,0 @@ -package system - -import ( - "os" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/yargevad/filepathx" -) - -// This collector collects arbitrary user files - -type FileCollector struct { - resourceManager *platform.ResourceManager -} - -func NewFileCollector(m *platform.ResourceManager) *FileCollector { - return &FileCollector{ - resourceManager: m, - } -} - -func (f *FileCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(f.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - filePattern, err := config.ReadStringValue(conf, "file_pattern") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - var frameErrors []error - // this util package supports recursive file matching e.g. /**/* - matches, err := filepathx.Glob(filePattern) - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "Invalid file_pattern \"%s\"", filePattern) - } - - if len(matches) == 0 { - frameErrors = append(frameErrors, errors.New("0 files match glob pattern")) - return &data.DiagnosticBundle{ - Errors: data.FrameErrors{Errors: frameErrors}, - }, nil - } - - var filePaths []string - for _, match := range matches { - fi, err := os.Stat(match) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to read file %s", match)) - } - if !fi.IsDir() { - log.Debug().Msgf("Collecting file %s", match) - filePaths = append(filePaths, match) - } - } - - frame := data.NewFileFrame("collection", filePaths) - - return &data.DiagnosticBundle{ - Errors: data.FrameErrors{Errors: frameErrors}, - Frames: map[string]data.Frame{ - "collection": frame, - }, - }, nil -} - -func (f *FileCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - } -} - -func (f *FileCollector) IsDefault() bool { - return false -} - -func (f *FileCollector) Description() string { - return "Allows collection of user specified files" -} - -// here we register the collector for use -func init() { - collectors.Register("file", func() (collectors.Collector, error) { - return &FileCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/system/file_test.go b/programs/diagnostics/internal/collectors/system/file_test.go deleted file mode 100644 index 5b1d5b3a92f..00000000000 --- a/programs/diagnostics/internal/collectors/system/file_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package system_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestFileConfiguration(t *testing.T) { - t.Run("correct configuration is returned for file collector", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - conf := fileCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.StringParam{}, conf.Params[0]) - filePattern, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.True(t, filePattern.Required()) - require.Equal(t, filePattern.Name(), "file_pattern") - require.Equal(t, "", filePattern.Value) - }) -} - -func TestFileCollect(t *testing.T) { - - t.Run("test filter patterns work", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - bundle, err := fileCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "../../../testdata/**/*.xml", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - }) - require.Nil(t, err) - require.Nil(t, bundle.Errors.Errors) - checkFileBundle(t, bundle, - []string{"../../../testdata/configs/include/xml/server-include.xml", - "../../../testdata/configs/include/xml/user-include.xml", - "../../../testdata/configs/xml/config.xml", - "../../../testdata/configs/xml/users.xml", - "../../../testdata/configs/xml/users.d/default-password.xml", - "../../../testdata/configs/yandex_xml/config.xml", - "../../../testdata/docker/admin.xml", - "../../../testdata/docker/custom.xml"}) - }) - - t.Run("invalid file patterns are detected", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - bundle, err := fileCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - }) - require.NotNil(t, err) - require.Equal(t, "parameter file_pattern is invalid - file_pattern cannot be empty", err.Error()) - require.Equal(t, &data.DiagnosticBundle{}, bundle) - }) - - t.Run("check empty matches are reported", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - bundle, err := fileCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "../../../testdata/**/*.random", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - }) - require.Nil(t, err) - require.Nil(t, bundle.Frames) - require.Len(t, bundle.Errors.Errors, 1) - require.Equal(t, "0 files match glob pattern", bundle.Errors.Errors[0].Error()) - }) - -} - -func checkFileBundle(t *testing.T, bundle *data.DiagnosticBundle, expectedFiles []string) { - require.NotNil(t, bundle) - require.Nil(t, bundle.Errors.Errors) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "collection") - dirFrame, ok := bundle.Frames["collection"].(data.FileFrame) - require.True(t, ok) - require.Equal(t, []string{"files"}, dirFrame.Columns()) - i := 0 - for { - values, ok, err := dirFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - file, ok := values[0].(data.SimpleFile) - require.True(t, ok) - require.Contains(t, expectedFiles, file.FilePath()) - i += 1 - } - require.Equal(t, len(expectedFiles), i) -} diff --git a/programs/diagnostics/internal/collectors/system/system.go b/programs/diagnostics/internal/collectors/system/system.go deleted file mode 100644 index 69d16f36b8b..00000000000 --- a/programs/diagnostics/internal/collectors/system/system.go +++ /dev/null @@ -1,235 +0,0 @@ -package system - -import ( - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/elastic/gosigar" - "github.com/jaypipes/ghw" - "github.com/matishsiao/goInfo" - "github.com/pkg/errors" -) - -// This collector collects the system overview - -type SystemCollector struct { - resourceManager *platform.ResourceManager -} - -func NewSystemCollector(m *platform.ResourceManager) *SystemCollector { - return &SystemCollector{ - resourceManager: m, - } -} - -func (sc *SystemCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - _, err := conf.ValidateConfig(sc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - frames := make(map[string]data.Frame) - var frameErrors []error - - frameErrors = addStatsToFrame(frames, frameErrors, "disks", getDisk) - frameErrors = addStatsToFrame(frames, frameErrors, "disk_usage", getDiskUsage) - - frameErrors = addStatsToFrame(frames, frameErrors, "memory", getMemory) - frameErrors = addStatsToFrame(frames, frameErrors, "memory_usage", getMemoryUsage) - - frameErrors = addStatsToFrame(frames, frameErrors, "cpu", getCPU) - //frameErrors = addStatsToFrame(frames, frameErrors, "cpu_usage", getCPUUsage) - - frameErrors = addStatsToFrame(frames, frameErrors, "processes", getProcessList) - - frameErrors = addStatsToFrame(frames, frameErrors, "os", getHostDetails) - - return &data.DiagnosticBundle{ - Frames: frames, - Errors: data.FrameErrors{ - Errors: frameErrors, - }, - }, err -} - -func addStatsToFrame(frames map[string]data.Frame, errors []error, name string, statFunc func() (data.MemoryFrame, error)) []error { - frame, err := statFunc() - if err != nil { - errors = append(errors, err) - } - frames[name] = frame - return errors -} - -func (sc *SystemCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{}, - } -} - -func (sc *SystemCollector) IsDefault() bool { - return true -} - -func getDisk() (data.MemoryFrame, error) { - block, err := ghw.Block() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list block storage") - } - var rows [][]interface{} - columns := []string{"name", "size", "physicalBlockSize", "driveType", "controller", "vendor", "model", "partitionName", "partitionSize", "mountPoint", "readOnly"} - for _, disk := range block.Disks { - for _, part := range disk.Partitions { - rows = append(rows, []interface{}{disk.Name, disk.SizeBytes, disk.PhysicalBlockSizeBytes, disk.DriveType, disk.StorageController, disk.Vendor, disk.Model, part.Name, part.SizeBytes, part.MountPoint, part.IsReadOnly}) - } - } - return data.NewMemoryFrame("disk_usage", columns, rows), nil -} - -func getDiskUsage() (data.MemoryFrame, error) { - fsList := gosigar.FileSystemList{} - err := fsList.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list filesystems for usage") - } - rows := make([][]interface{}, len(fsList.List)) - columns := []string{"filesystem", "size", "used", "avail", "use%", "mounted on"} - for i, fs := range fsList.List { - dirName := fs.DirName - usage := gosigar.FileSystemUsage{} - err = usage.Get(dirName) - if err == nil { - rows[i] = []interface{}{fs.DevName, usage.Total, usage.Used, usage.Avail, usage.UsePercent(), dirName} - } else { - // we try to output something - rows[i] = []interface{}{fs.DevName, 0, 0, 0, 0, dirName} - } - } - return data.NewMemoryFrame("disk_usage", columns, rows), nil -} - -func getMemory() (data.MemoryFrame, error) { - memory, err := ghw.Memory() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to read memory") - } - columns := []string{"totalPhysical", "totalUsable", "supportedPageSizes"} - rows := make([][]interface{}, 1) - rows[0] = []interface{}{memory.TotalPhysicalBytes, memory.TotalUsableBytes, memory.SupportedPageSizes} - return data.NewMemoryFrame("memory", columns, rows), nil -} - -func getMemoryUsage() (data.MemoryFrame, error) { - mem := gosigar.Mem{} - swap := gosigar.Swap{} - - err := mem.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to read memory usage") - } - - err = swap.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to read swap") - } - - columns := []string{"type", "total", "used", "free"} - rows := make([][]interface{}, 3) - - rows[0] = []interface{}{"mem", mem.Total, mem.Used, mem.Free} - rows[1] = []interface{}{"buffers/cache", 0, mem.ActualUsed, mem.ActualFree} - rows[2] = []interface{}{"swap", swap.Total, swap.Used, swap.Free} - return data.NewMemoryFrame("memory_usage", columns, rows), nil - -} - -func getCPU() (data.MemoryFrame, error) { - cpu, err := ghw.CPU() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list cpus") - } - columns := []string{"processor", "vendor", "model", "core", "numThreads", "logical", "capabilities"} - var rows [][]interface{} - for _, proc := range cpu.Processors { - for _, core := range proc.Cores { - rows = append(rows, []interface{}{proc.ID, proc.Vendor, proc.Model, core.ID, core.NumThreads, core.LogicalProcessors, strings.Join(proc.Capabilities, " ")}) - } - } - return data.NewMemoryFrame("cpu", columns, rows), nil -} - -// this gets cpu usage vs a listing of arch etc - see getCPU(). This needs successive values as its ticks - not currently used -// see https://github.com/elastic/beats/blob/master/metricbeat/internal/metrics/cpu/metrics.go#L131 for inspiration -//nolint -func getCPUUsage() (data.MemoryFrame, error) { - cpuList := gosigar.CpuList{} - err := cpuList.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list cpus for usage") - } - columns := []string{"sys", "nice", "stolen", "irq", "idle", "softIrq", "user", "wait", "total"} - rows := make([][]interface{}, len(cpuList.List), len(cpuList.List)) - for i, cpu := range cpuList.List { - rows[i] = []interface{}{cpu.Sys, cpu.Nice, cpu.Stolen, cpu.Irq, cpu.Idle, cpu.SoftIrq, cpu.User, cpu.Wait, cpu.Total()} - } - return data.NewMemoryFrame("cpu_usage", columns, rows), nil -} - -func getProcessList() (data.MemoryFrame, error) { - pidList := gosigar.ProcList{} - err := pidList.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list processes") - } - columns := []string{"pid", "ppid", "stime", "time", "rss", "size", "faults", "minorFaults", "majorFaults", "user", "state", "priority", "nice", "command"} - rows := make([][]interface{}, len(pidList.List)) - for i, pid := range pidList.List { - state := gosigar.ProcState{} - mem := gosigar.ProcMem{} - time := gosigar.ProcTime{} - args := gosigar.ProcArgs{} - if err := state.Get(pid); err != nil { - continue - } - if err := mem.Get(pid); err != nil { - continue - } - if err := time.Get(pid); err != nil { - continue - } - if err := args.Get(pid); err != nil { - continue - } - rows[i] = []interface{}{pid, state.Ppid, time.FormatStartTime(), time.FormatTotal(), mem.Resident, mem.Size, - mem.PageFaults, mem.MinorFaults, mem.MajorFaults, state.Username, state.State, state.Priority, state.Nice, - strings.Join(args.List, " ")} - } - return data.NewMemoryFrame("process_list", columns, rows), nil -} - -func getHostDetails() (data.MemoryFrame, error) { - gi, err := goInfo.GetInfo() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to get host summary") - } - columns := []string{"hostname", "os", "goOs", "cpus", "core", "kernel", "platform"} - rows := [][]interface{}{ - {gi.Hostname, gi.OS, gi.GoOS, gi.CPUs, gi.Core, gi.Kernel, gi.Platform}, - } - return data.NewMemoryFrame("os", columns, rows), nil -} - -func (sc *SystemCollector) Description() string { - return "Collects summary OS and hardware statistics for the host" -} - -// here we register the collector for use -func init() { - collectors.Register("system", func() (collectors.Collector, error) { - return &SystemCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/system/system_test.go b/programs/diagnostics/internal/collectors/system/system_test.go deleted file mode 100644 index fb1e16bd1ed..00000000000 --- a/programs/diagnostics/internal/collectors/system/system_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package system_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestSystemConfiguration(t *testing.T) { - t.Run("correct configuration is returned for system collector", func(t *testing.T) { - systemCollector := system.NewSystemCollector(&platform.ResourceManager{}) - conf := systemCollector.Configuration() - require.Len(t, conf.Params, 0) - require.Equal(t, []config.ConfigParam{}, conf.Params) - }) -} - -func TestSystemCollect(t *testing.T) { - t.Run("test default system collection", func(t *testing.T) { - systemCollector := system.NewSystemCollector(&platform.ResourceManager{}) - diagSet, err := systemCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 7) - require.Contains(t, diagSet.Frames, "disks") - require.Contains(t, diagSet.Frames, "disk_usage") - require.Contains(t, diagSet.Frames, "memory") - require.Contains(t, diagSet.Frames, "memory_usage") - require.Contains(t, diagSet.Frames, "cpu") - require.Contains(t, diagSet.Frames, "processes") - require.Contains(t, diagSet.Frames, "os") - // responses here will vary depending on platform - mocking seems excessive so we test we have some data - // disks - require.Equal(t, []string{"name", "size", "physicalBlockSize", "driveType", "controller", "vendor", "model", "partitionName", "partitionSize", "mountPoint", "readOnly"}, diagSet.Frames["disks"].Columns()) - diskFrames, err := countFrameRows(diagSet, "disks") - require.Greater(t, diskFrames, 0) - require.Nil(t, err) - // disk usage - require.Equal(t, []string{"filesystem", "size", "used", "avail", "use%", "mounted on"}, diagSet.Frames["disk_usage"].Columns()) - diskUsageFrames, err := countFrameRows(diagSet, "disk_usage") - require.Greater(t, diskUsageFrames, 0) - require.Nil(t, err) - // memory - require.Equal(t, []string{"totalPhysical", "totalUsable", "supportedPageSizes"}, diagSet.Frames["memory"].Columns()) - memoryFrames, err := countFrameRows(diagSet, "memory") - require.Greater(t, memoryFrames, 0) - require.Nil(t, err) - // memory_usage - require.Equal(t, []string{"type", "total", "used", "free"}, diagSet.Frames["memory_usage"].Columns()) - memoryUsageFrames, err := countFrameRows(diagSet, "memory_usage") - require.Greater(t, memoryUsageFrames, 0) - require.Nil(t, err) - // cpu - require.Equal(t, []string{"processor", "vendor", "model", "core", "numThreads", "logical", "capabilities"}, diagSet.Frames["cpu"].Columns()) - cpuFrames, err := countFrameRows(diagSet, "cpu") - require.Greater(t, cpuFrames, 0) - require.Nil(t, err) - // processes - require.Equal(t, []string{"pid", "ppid", "stime", "time", "rss", "size", "faults", "minorFaults", "majorFaults", "user", "state", "priority", "nice", "command"}, diagSet.Frames["processes"].Columns()) - processesFrames, err := countFrameRows(diagSet, "processes") - require.Greater(t, processesFrames, 0) - require.Nil(t, err) - // os - require.Equal(t, []string{"hostname", "os", "goOs", "cpus", "core", "kernel", "platform"}, diagSet.Frames["os"].Columns()) - osFrames, err := countFrameRows(diagSet, "os") - require.Greater(t, osFrames, 0) - require.Nil(t, err) - }) -} - -func countFrameRows(diagSet *data.DiagnosticBundle, frameName string) (int, error) { - frame := diagSet.Frames[frameName] - i := 0 - for { - _, ok, err := frame.Next() - if !ok { - return i, err - } - if err != nil { - return i, err - } - i++ - } -} diff --git a/programs/diagnostics/internal/outputs/file/simple.go b/programs/diagnostics/internal/outputs/file/simple.go deleted file mode 100644 index 63847b3addd..00000000000 --- a/programs/diagnostics/internal/outputs/file/simple.go +++ /dev/null @@ -1,344 +0,0 @@ -package file - -import ( - "context" - "encoding/csv" - "fmt" - "os" - "path" - "path/filepath" - "strconv" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/mholt/archiver/v4" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -const OutputName = "simple" - -type SubFolderGenerator func() string - -type SimpleOutput struct { - // mainly used for testing to make sub folder deterministic - which it won't be by default as it uses a timestamp - FolderGenerator SubFolderGenerator -} - -func (o SimpleOutput) Write(id string, bundles map[string]*data.DiagnosticBundle, conf config.Configuration) (data.FrameErrors, error) { - conf, err := conf.ValidateConfig(o.Configuration()) - if err != nil { - return data.FrameErrors{}, err - } - directory, err := config.ReadStringValue(conf, "directory") - if err != nil { - return data.FrameErrors{}, err - } - directory, err = getWorkingDirectory(directory) - if err != nil { - return data.FrameErrors{}, err - } - subFolder := strconv.FormatInt(utils.MakeTimestamp(), 10) - if o.FolderGenerator != nil { - subFolder = o.FolderGenerator() - } - skipArchive, err := config.ReadBoolValue(conf, "skip_archive") - if err != nil { - return data.FrameErrors{}, err - } - - outputDir := filepath.Join(directory, id, subFolder) - log.Info().Msgf("creating bundle in %s", outputDir) - if err := os.MkdirAll(outputDir, os.ModePerm); err != nil { - return data.FrameErrors{}, err - } - frameErrors := data.FrameErrors{} - var filePaths []string - for name := range bundles { - bundlePaths, frameError := writeDiagnosticBundle(name, bundles[name], outputDir) - filePaths = append(filePaths, bundlePaths...) - frameErrors.Errors = append(frameErrors.Errors, frameError.Errors...) - } - log.Info().Msg("bundle created") - if !skipArchive { - archiveFilename := filepath.Join(directory, id, fmt.Sprintf("%s.tar.gz", subFolder)) - log.Info().Msgf("compressing bundle to %s", archiveFilename) - // produce a map containing the input paths to the archive paths - we preserve the output directory and hierarchy - archiveMap := createArchiveMap(filePaths, directory) - if err := createArchive(archiveFilename, archiveMap); err != nil { - return frameErrors, err - } - // we delete the original directory leaving just the archive behind - if err := os.RemoveAll(outputDir); err != nil { - return frameErrors, err - } - log.Info().Msgf("archive ready at: %s ", archiveFilename) - } - return frameErrors, nil -} - -func writeDiagnosticBundle(name string, diag *data.DiagnosticBundle, baseDir string) ([]string, data.FrameErrors) { - diagDir := filepath.Join(baseDir, name) - if err := os.MkdirAll(diagDir, os.ModePerm); err != nil { - return nil, data.FrameErrors{Errors: []error{ - errors.Wrapf(err, "unable to create directory for %s", name), - }} - } - frameErrors := data.FrameErrors{} - var filePaths []string - for frameId, frame := range diag.Frames { - fFilePath, errs := writeFrame(frameId, frame, diagDir) - filePaths = append(filePaths, fFilePath...) - if len(errs) > 0 { - // it would be nice if we could wrap this list of errors into something formal but this logs well - frameErrors.Errors = append(frameErrors.Errors, fmt.Errorf("unable to write frame %s for %s", frameId, name)) - frameErrors.Errors = append(frameErrors.Errors, errs...) - } - } - return filePaths, frameErrors -} - -func writeFrame(frameId string, frame data.Frame, baseDir string) ([]string, []error) { - switch f := frame.(type) { - case data.DatabaseFrame: - return writeDatabaseFrame(frameId, f, baseDir) - case data.ConfigFileFrame: - return writeConfigFrame(frameId, f, baseDir) - case data.DirectoryFileFrame: - return processDirectoryFileFrame(frameId, f, baseDir) - case data.FileFrame: - return processFileFrame(frameId, f, baseDir) - case data.HierarchicalFrame: - return writeHierarchicalFrame(frameId, f, baseDir) - default: - // for now our data frame writer supports all frames - return writeDatabaseFrame(frameId, frame, baseDir) - } -} - -func writeHierarchicalFrame(frameId string, frame data.HierarchicalFrame, baseDir string) ([]string, []error) { - filePaths, errs := writeFrame(frameId, frame.DataFrame, baseDir) - for _, subFrame := range frame.SubFrames { - subDir := filepath.Join(baseDir, subFrame.Name()) - if err := os.MkdirAll(subDir, os.ModePerm); err != nil { - errs = append(errs, err) - continue - } - subPaths, subErrs := writeFrame(subFrame.Name(), subFrame, subDir) - filePaths = append(filePaths, subPaths...) - errs = append(errs, subErrs...) - } - return filePaths, errs -} - -func writeDatabaseFrame(frameId string, frame data.Frame, baseDir string) ([]string, []error) { - frameFilePath := filepath.Join(baseDir, fmt.Sprintf("%s.csv", frameId)) - var errs []error - f, err := os.Create(frameFilePath) - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - defer f.Close() - w := csv.NewWriter(f) - defer w.Flush() - if err := w.Write(frame.Columns()); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to write columns for frame %s", frameId)) - return []string{}, errs - } - // we don't collect an error for every line here like configs and logs - could mean a lot of unnecessary noise - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to read frame %s", frameId)) - return []string{}, errs - } - if !ok { - return []string{frameFilePath}, errs - } - sValues := make([]string, len(values)) - for i, value := range values { - sValues[i] = fmt.Sprintf("%v", value) - } - if err := w.Write(sValues); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to write row for frame %s", frameId)) - return []string{}, errs - } - } -} - -func writeConfigFrame(frameId string, frame data.ConfigFileFrame, baseDir string) ([]string, []error) { - var errs []error - frameDirectory := filepath.Join(baseDir, frameId) - if err := os.MkdirAll(frameDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - // this holds our files included - includesDirectory := filepath.Join(frameDirectory, "includes") - if err := os.MkdirAll(includesDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create includes directory for frame %s", frameId)) - return []string{}, errs - } - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, err) - return []string{frameDirectory}, errs - } - if !ok { - return []string{frameDirectory}, errs - } - configFile := values[0].(data.ConfigFile) - if !configFile.IsIncluded() { - relPath := strings.TrimPrefix(configFile.FilePath(), frame.Directory) - destPath := path.Join(frameDirectory, relPath) - if err = configFile.Copy(destPath, true); err != nil { - errs = append(errs, errors.Wrapf(err, "Unable to copy file %s", configFile.FilePath())) - } - } else { - // include files could be anywhere - potentially multiple with the same name. We thus, recreate the directory - // hierarchy under includes to avoid collisions - destPath := path.Join(includesDirectory, configFile.FilePath()) - if err = configFile.Copy(destPath, true); err != nil { - errs = append(errs, errors.Wrapf(err, "Unable to copy file %s", configFile.FilePath())) - } - } - - } -} - -func processDirectoryFileFrame(frameId string, frame data.DirectoryFileFrame, baseDir string) ([]string, []error) { - var errs []error - // each set of files goes under its own directory to preserve grouping - frameDirectory := filepath.Join(baseDir, frameId) - if err := os.MkdirAll(frameDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, err) - return []string{frameDirectory}, errs - } - if !ok { - return []string{frameDirectory}, errs - } - file := values[0].(data.SimpleFile) - relPath := strings.TrimPrefix(file.FilePath(), frame.Directory) - destPath := path.Join(frameDirectory, relPath) - - if err = file.Copy(destPath, true); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to copy file %s for frame %s", file, frameId)) - } - } -} - -func processFileFrame(frameId string, frame data.FileFrame, baseDir string) ([]string, []error) { - var errs []error - frameDirectory := filepath.Join(baseDir, frameId) - if err := os.MkdirAll(frameDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, err) - } - if !ok { - return []string{frameDirectory}, errs - } - file := values[0].(data.SimpleFile) - // we need an absolute path to preserve the directory hierarchy - dir, err := filepath.Abs(filepath.Dir(file.FilePath())) - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to determine dir for %s", file.FilePath())) - } - outputDir := filepath.Join(frameDirectory, dir) - if _, err := os.Stat(outputDir); os.IsNotExist(err) { - if err := os.MkdirAll(outputDir, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for %s", file.FilePath())) - } else { - outputPath := filepath.Join(outputDir, filepath.Base(file.FilePath())) - err = file.Copy(outputPath, false) - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to copy file %s", file.FilePath())) - } - } - } - } -} - -func getWorkingDirectory(path string) (string, error) { - if !filepath.IsAbs(path) { - workingPath, err := os.Getwd() - if err != nil { - return "", err - } - return filepath.Join(workingPath, path), nil - } - return path, nil -} - -func createArchiveMap(filePaths []string, prefix string) map[string]string { - archiveMap := make(map[string]string) - for _, path := range filePaths { - archiveMap[path] = strings.TrimPrefix(path, prefix) - } - return archiveMap -} - -func createArchive(outputFile string, filePaths map[string]string) error { - files, err := archiver.FilesFromDisk(nil, filePaths) - if err != nil { - return err - } - out, err := os.Create(outputFile) - if err != nil { - return err - } - defer out.Close() - format := archiver.CompressedArchive{ - Compression: archiver.Gz{}, - Archival: archiver.Tar{}, - } - err = format.Archive(context.Background(), out, files) - return err -} - -func (o SimpleOutput) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "./", - Param: config.NewParam("directory", "Directory in which to create dump. Defaults to the current directory.", false), - }, - config.StringOptions{ - Value: "csv", - // TODO: add tsv and others here later - Options: []string{"csv"}, - Param: config.NewParam("format", "Format of exported files", false), - }, - config.BoolParam{ - Value: false, - Param: config.NewParam("skip_archive", "Don't compress output to an archive", false), - }, - }, - } -} - -func (o SimpleOutput) Description() string { - return "Writes out the diagnostic bundle as files in a structured directory, optionally producing a compressed archive." -} - -// here we register the output for use -func init() { - outputs.Register(OutputName, func() (outputs.Output, error) { - return SimpleOutput{}, nil - }) -} diff --git a/programs/diagnostics/internal/outputs/file/simple_test.go b/programs/diagnostics/internal/outputs/file/simple_test.go deleted file mode 100644 index 471a1c70cc1..00000000000 --- a/programs/diagnostics/internal/outputs/file/simple_test.go +++ /dev/null @@ -1,468 +0,0 @@ -package file_test - -import ( - "bufio" - "encoding/xml" - "fmt" - "io" - "os" - "path" - "strings" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -var clusterFrame = test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - {"events", 2, 1, 1, "dalem-local-clickhouse-blue-2", "192.168.144.4", 9001, 1, "default", "", 0, 0, 0}, - {"events", 3, 1, 1, "dalem-local-clickhouse-blue-3", "192.168.144.3", 9002, 1, "default", "", 0, 0, 0}, - }, -) - -var diskFrame = test.NewFakeDataFrame("disks", []string{"name", "path", "free_space", "total_space", "keep_free_space", "type"}, - [][]interface{}{ - {"default", "/var/lib/clickhouse", 1729659346944, 1938213220352, "", "local"}, - }, -) - -var userFrame = test.NewFakeDataFrame("users", []string{"name", "id", "storage", "auth_type", "auth_params", "host_ip", "host_names", "host_names_regexp", "host_names_like"}, - [][]interface{}{ - {"default", "94309d50-4f52-5250-31bd-74fecac179db,users.xml,plaintext_password", "sha256_password", []string{"::0"}, []string{}, []string{}, []string{}}, - }, -) - -func TestConfiguration(t *testing.T) { - t.Run("correct configuration is returned", func(t *testing.T) { - output := file.SimpleOutput{} - conf := output.Configuration() - require.Len(t, conf.Params, 3) - // check first directory param - require.IsType(t, config.StringParam{}, conf.Params[0]) - directory, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, directory.Required()) - require.Equal(t, "directory", directory.Name()) - require.Equal(t, "./", directory.Value) - // check second format param - require.IsType(t, config.StringOptions{}, conf.Params[1]) - format, ok := conf.Params[1].(config.StringOptions) - require.True(t, ok) - require.False(t, format.Required()) - require.Equal(t, "format", format.Name()) - require.Equal(t, "csv", format.Value) - require.Equal(t, []string{"csv"}, format.Options) - // check third format compress - require.IsType(t, config.BoolParam{}, conf.Params[2]) - skipArchive, ok := conf.Params[2].(config.BoolParam) - require.True(t, ok) - require.False(t, format.Required()) - require.False(t, skipArchive.Value) - }) -} - -func TestWrite(t *testing.T) { - bundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "systemB": { - Frames: map[string]data.Frame{ - "user": userFrame, - }, - }, - } - t.Run("test we can write simple diagnostic sets", func(t *testing.T) { - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", bundles, configuration) - require.Nil(t, err) - require.Equal(t, data.FrameErrors{}, frameErrors) - clusterFile := path.Join(tempDir, "test", "test", "systemA", "cluster.csv") - diskFile := path.Join(tempDir, "test", "test", "systemA", "disk.csv") - userFile := path.Join(tempDir, "test", "test", "systemB", "user.csv") - require.FileExists(t, clusterFile) - require.FileExists(t, diskFile) - require.FileExists(t, userFile) - diskLines, err := readFileLines(diskFile) - require.Nil(t, err) - require.Len(t, diskLines, 2) - usersLines, err := readFileLines(userFile) - require.Nil(t, err) - require.Len(t, usersLines, 2) - clusterLines, err := readFileLines(clusterFile) - require.Nil(t, err) - require.Len(t, clusterLines, 4) - require.Equal(t, strings.Join(clusterFrame.ColumnNames, ","), clusterLines[0]) - require.Equal(t, "events,1,1,1,dalem-local-clickhouse-blue-1,192.168.144.2,9000,1,default,,0,0,0", clusterLines[1]) - require.Equal(t, "events,2,1,1,dalem-local-clickhouse-blue-2,192.168.144.4,9001,1,default,,0,0,0", clusterLines[2]) - require.Equal(t, "events,3,1,1,dalem-local-clickhouse-blue-3,192.168.144.3,9002,1,default,,0,0,0", clusterLines[3]) - resetFrames() - }) - - t.Run("test invalid parameter", func(t *testing.T) { - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - config.StringOptions{ - Value: "random", - Options: []string{"csv"}, - // TODO: add tsv and others here later - Param: config.NewParam("format", "Format of exported files", false), - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip compressed archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", bundles, configuration) - require.Equal(t, data.FrameErrors{}, frameErrors) - require.NotNil(t, err) - require.Equal(t, "parameter format is invalid - random is not a valid value for format - [csv]", err.Error()) - resetFrames() - }) - - t.Run("test compression", func(t *testing.T) { - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", bundles, configuration) - require.Nil(t, err) - require.Equal(t, data.FrameErrors{}, frameErrors) - archiveFileName := path.Join(tempDir, "test", "test.tar.gz") - fi, err := os.Stat(archiveFileName) - require.Nil(t, err) - require.FileExists(t, archiveFileName) - // compression will vary so lets test range - require.Greater(t, int64(600), fi.Size()) - require.Less(t, int64(200), fi.Size()) - outputFolder := path.Join(tempDir, "test", "test") - // check the folder doesn't exist and is cleaned up - require.NoFileExists(t, outputFolder) - resetFrames() - }) - - t.Run("test support for directory frames", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - files := createRandomFiles(tempDir, 5) - dirFrame, errs := data.NewFileDirectoryFrame(tempDir, []string{"*.log"}) - require.Empty(t, errs) - fileBundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "config": { - Frames: map[string]data.Frame{ - "logs": dirFrame, - }, - }, - } - destDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: destDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", fileBundles, configuration) - require.Nil(t, err) - require.NotNil(t, frameErrors) - - // test the usual frames still work - clusterFile := path.Join(destDir, "test", "test", "systemA", "cluster.csv") - diskFile := path.Join(destDir, "test", "test", "systemA", "disk.csv") - require.FileExists(t, clusterFile) - require.FileExists(t, diskFile) - diskLines, err := readFileLines(diskFile) - require.Nil(t, err) - require.Len(t, diskLines, 2) - clusterLines, err := readFileLines(clusterFile) - require.Nil(t, err) - require.Len(t, clusterLines, 4) - require.Equal(t, strings.Join(clusterFrame.ColumnNames, ","), clusterLines[0]) - require.Equal(t, "events,1,1,1,dalem-local-clickhouse-blue-1,192.168.144.2,9000,1,default,,0,0,0", clusterLines[1]) - require.Equal(t, "events,2,1,1,dalem-local-clickhouse-blue-2,192.168.144.4,9001,1,default,,0,0,0", clusterLines[2]) - require.Equal(t, "events,3,1,1,dalem-local-clickhouse-blue-3,192.168.144.3,9002,1,default,,0,0,0", clusterLines[3]) - //test our directory frame - for _, filepath := range files { - // check they were copied - subPath := strings.TrimPrefix(filepath, tempDir) - // path here will be //test>/config/logs/ - newPath := path.Join(destDir, "test", "test", "config", "logs", subPath) - require.FileExists(t, newPath) - } - resetFrames() - }) - - t.Run("test support for config frames", func(t *testing.T) { - xmlConfig := data.XmlConfig{ - XMLName: xml.Name{}, - Clickhouse: data.XmlLoggerConfig{ - XMLName: xml.Name{}, - ErrorLog: "/var/log/clickhouse-server/clickhouse-server.err.log", - Log: "/var/log/clickhouse-server/clickhouse-server.log", - }, - IncludeFrom: "", - } - tempDir := t.TempDir() - confDir := path.Join(tempDir, "conf") - // create an includes file - includesDir := path.Join(tempDir, "includes") - err := os.MkdirAll(includesDir, os.ModePerm) - require.Nil(t, err) - includesPath := path.Join(includesDir, "random.xml") - includesFile, err := os.Create(includesPath) - require.Nil(t, err) - xmlWriter := io.Writer(includesFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - err = enc.Encode(xmlConfig) - require.Nil(t, err) - // create 5 temporary config files - files := make([]string, 5) - // set the includes - xmlConfig.IncludeFrom = includesPath - for i := 0; i < 5; i++ { - // we want to check hierarchies are preserved so create a simple folder for each file - fileDir := path.Join(confDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.xml", i)) - files[i] = filepath - xmlFile, err := os.Create(filepath) - require.Nil(t, err) - // write a little xml so its valid - xmlWriter := io.Writer(xmlFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - err = enc.Encode(xmlConfig) - require.Nil(t, err) - } - configFrame, errs := data.NewConfigFileFrame(confDir) - require.Empty(t, errs) - fileBundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "config": { - Frames: map[string]data.Frame{ - "user_specified": configFrame, - }, - }, - } - destDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: destDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", fileBundles, configuration) - require.Nil(t, err) - require.NotNil(t, frameErrors) - require.Empty(t, frameErrors.Errors) - //test our config frame - for _, filepath := range files { - // check they were copied - subPath := strings.TrimPrefix(filepath, confDir) - // path here will be //test>/config/user_specified/file - newPath := path.Join(destDir, "test", "test", "config", "user_specified", subPath) - require.FileExists(t, newPath) - } - // check our includes file exits - // path here will be //test>/config/user_specified/file/includes - require.FileExists(t, path.Join(destDir, "test", "test", "config", "user_specified", "includes", includesPath)) - resetFrames() - }) - - t.Run("test support for file frames", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - files := createRandomFiles(tempDir, 5) - fileFrame := data.NewFileFrame("collection", files) - fileBundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "file": { - Frames: map[string]data.Frame{ - "collection": fileFrame, - }, - }, - } - destDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: destDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", fileBundles, configuration) - require.Nil(t, err) - require.NotNil(t, frameErrors) - //test our directory frame - for _, filepath := range files { - // path here will be //test>/file/collection/ - newPath := path.Join(destDir, "test", "test", "file", "collection", filepath) - require.FileExists(t, newPath) - } - resetFrames() - }) - - t.Run("test support for hierarchical frames", func(t *testing.T) { - bottomFrame := data.NewHierarchicalFrame("bottomLevel", userFrame, []data.HierarchicalFrame{}) - middleFrame := data.NewHierarchicalFrame("middleLevel", diskFrame, []data.HierarchicalFrame{bottomFrame}) - topFrame := data.NewHierarchicalFrame("topLevel", clusterFrame, []data.HierarchicalFrame{middleFrame}) - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - hierarchicalBundle := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "topLevel": topFrame, - }, - }, - } - frameErrors, err := output.Write("test", hierarchicalBundle, configuration) - require.Nil(t, err) - require.Equal(t, data.FrameErrors{}, frameErrors) - topFile := path.Join(tempDir, "test", "test", "systemA", "topLevel.csv") - middleFile := path.Join(tempDir, "test", "test", "systemA", "middleLevel", "middleLevel.csv") - bottomFile := path.Join(tempDir, "test", "test", "systemA", "middleLevel", "bottomLevel", "bottomLevel.csv") - require.FileExists(t, topFile) - require.FileExists(t, middleFile) - require.FileExists(t, bottomFile) - topLines, err := readFileLines(topFile) - require.Nil(t, err) - require.Len(t, topLines, 4) - middleLines, err := readFileLines(middleFile) - require.Nil(t, err) - require.Len(t, middleLines, 2) - bottomLines, err := readFileLines(bottomFile) - require.Nil(t, err) - require.Len(t, bottomLines, 2) - require.Equal(t, strings.Join(clusterFrame.ColumnNames, ","), topLines[0]) - require.Equal(t, "events,1,1,1,dalem-local-clickhouse-blue-1,192.168.144.2,9000,1,default,,0,0,0", topLines[1]) - require.Equal(t, "events,2,1,1,dalem-local-clickhouse-blue-2,192.168.144.4,9001,1,default,,0,0,0", topLines[2]) - require.Equal(t, "events,3,1,1,dalem-local-clickhouse-blue-3,192.168.144.3,9002,1,default,,0,0,0", topLines[3]) - resetFrames() - }) -} - -func createRandomFiles(tempDir string, num int) []string { - files := make([]string, num) - for i := 0; i < 5; i++ { - // we want to check hierarchies are preserved so create a simple folder for each file - fileDir := path.Join(tempDir, fmt.Sprintf("%d", i)) - os.MkdirAll(fileDir, os.ModePerm) //nolint:errcheck - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.log", i)) - files[i] = filepath - os.Create(filepath) //nolint:errcheck - } - return files -} - -func resetFrames() { - clusterFrame.Reset() - userFrame.Reset() - diskFrame.Reset() -} - -func readFileLines(filename string) ([]string, error) { - file, err := os.Open(filename) - if err != nil { - return nil, err - } - defer file.Close() - - var lines []string - scanner := bufio.NewScanner(file) - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - return lines, scanner.Err() -} - -func staticFolderName() string { - return "test" -} diff --git a/programs/diagnostics/internal/outputs/registry.go b/programs/diagnostics/internal/outputs/registry.go deleted file mode 100644 index 0187cd9105d..00000000000 --- a/programs/diagnostics/internal/outputs/registry.go +++ /dev/null @@ -1,67 +0,0 @@ -package outputs - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -type Output interface { - Write(id string, bundles map[string]*data.DiagnosticBundle, config config.Configuration) (data.FrameErrors, error) - Configuration() config.Configuration - Description() string - // TODO: we will need to implement this for the convert function - //Read(config config.Configuration) (data.DiagnosticBundle, error) -} - -// Register can be called from init() on an output in this package -// It will automatically be added to the Outputs map to be called externally -func Register(name string, output OutputFactory) { - // names must be unique - if _, ok := Outputs[name]; ok { - log.Error().Msgf("More than 1 output is trying to register under the name %s. Names must be unique.", name) - } - Outputs[name] = output -} - -// OutputFactory lets us use a closure to get instances of the output struct -type OutputFactory func() (Output, error) - -var Outputs = map[string]OutputFactory{} - -func GetOutputNames() []string { - outputs := make([]string, len(Outputs)) - i := 0 - for k := range Outputs { - outputs[i] = k - i++ - } - return outputs -} - -func GetOutputByName(name string) (Output, error) { - if outputFactory, ok := Outputs[name]; ok { - //do something here - output, err := outputFactory() - if err != nil { - return nil, errors.Wrapf(err, "output %s could not be initialized", name) - } - return output, nil - } - return nil, fmt.Errorf("%s is not a valid output name", name) -} - -func BuildConfigurationOptions() (map[string]config.Configuration, error) { - configurations := make(map[string]config.Configuration) - for name, collectorFactory := range Outputs { - output, err := collectorFactory() - if err != nil { - return nil, errors.Wrapf(err, "output %s could not be initialized", name) - } - configurations[name] = output.Configuration() - } - return configurations, nil -} diff --git a/programs/diagnostics/internal/outputs/registry_test.go b/programs/diagnostics/internal/outputs/registry_test.go deleted file mode 100644 index ba8408e5a59..00000000000 --- a/programs/diagnostics/internal/outputs/registry_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package outputs_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" - "github.com/stretchr/testify/require" -) - -func TestGetOutputNames(t *testing.T) { - t.Run("can get all output names", func(t *testing.T) { - outputNames := outputs.GetOutputNames() - require.ElementsMatch(t, []string{"simple", "report"}, outputNames) - }) - -} - -func TestGetOutputByName(t *testing.T) { - - t.Run("can get output by name", func(t *testing.T) { - output, err := outputs.GetOutputByName("simple") - require.Nil(t, err) - require.Equal(t, file.SimpleOutput{}, output) - }) - - t.Run("fails on non existing output", func(t *testing.T) { - output, err := outputs.GetOutputByName("random") - require.NotNil(t, err) - require.Equal(t, "random is not a valid output name", err.Error()) - require.Nil(t, output) - }) -} - -func TestBuildConfigurationOptions(t *testing.T) { - - t.Run("can get all output configurations", func(t *testing.T) { - outputs, err := outputs.BuildConfigurationOptions() - require.Nil(t, err) - require.Len(t, outputs, 2) - require.Contains(t, outputs, "simple") - require.Contains(t, outputs, "report") - }) -} diff --git a/programs/diagnostics/internal/outputs/terminal/report.go b/programs/diagnostics/internal/outputs/terminal/report.go deleted file mode 100644 index 8337f542457..00000000000 --- a/programs/diagnostics/internal/outputs/terminal/report.go +++ /dev/null @@ -1,284 +0,0 @@ -package terminal - -import ( - "bufio" - "fmt" - "os" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/olekukonko/tablewriter" - "github.com/pkg/errors" -) - -const OutputName = "report" - -type ReportOutput struct { -} - -func (r ReportOutput) Write(id string, bundles map[string]*data.DiagnosticBundle, conf config.Configuration) (data.FrameErrors, error) { - conf, err := conf.ValidateConfig(r.Configuration()) - if err != nil { - return data.FrameErrors{}, err - } - format, err := config.ReadStringOptionsValue(conf, "format") - if err != nil { - return data.FrameErrors{}, err - } - nonInteractive, err := config.ReadBoolValue(conf, "continue") - if err != nil { - return data.FrameErrors{}, err - } - maxRows, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return data.FrameErrors{}, err - } - maxColumns, err := config.ReadIntValue(conf, "column_limit") - if err != nil { - return data.FrameErrors{}, err - } - frameErrors := data.FrameErrors{} - for name := range bundles { - frameError := printDiagnosticBundle(name, bundles[name], format, !nonInteractive, int(maxRows), int(maxColumns)) - frameErrors.Errors = append(frameErrors.Errors, frameError.Errors...) - } - return data.FrameErrors{}, nil -} - -func printDiagnosticBundle(name string, diag *data.DiagnosticBundle, format string, interactive bool, maxRows, maxColumns int) data.FrameErrors { - frameErrors := data.FrameErrors{} - for frameId, frame := range diag.Frames { - printFrameHeader(fmt.Sprintf("%s.%s", name, frameId)) - err := printFrame(frame, format, maxRows, maxColumns) - if err != nil { - frameErrors.Errors = append(frameErrors.Errors, err) - } - if interactive { - err := waitForEnter() - if err != nil { - frameErrors.Errors = append(frameErrors.Errors, err) - } - } - } - return frameErrors -} - -func waitForEnter() error { - fmt.Println("Press the Enter Key to view the next frame report") - for { - consoleReader := bufio.NewReaderSize(os.Stdin, 1) - input, err := consoleReader.ReadByte() - if err != nil { - return errors.New("Unable to read user input") - } - if input == 3 { - //ctl +c - fmt.Println("Exiting...") - os.Exit(0) - } - if input == 10 { - return nil - } - } -} - -func printFrame(frame data.Frame, format string, maxRows, maxColumns int) error { - switch f := frame.(type) { - case data.DatabaseFrame: - return printDatabaseFrame(f, format, maxRows, maxColumns) - case data.ConfigFileFrame: - return printConfigFrame(f, format) - case data.DirectoryFileFrame: - return printDirectoryFileFrame(f, format, maxRows) - case data.HierarchicalFrame: - return printHierarchicalFrame(f, format, maxRows, maxColumns) - default: - // for now our data frame writer supports all frames - return printDatabaseFrame(f, format, maxRows, maxColumns) - } -} - -func createTable(format string) *tablewriter.Table { - table := tablewriter.NewWriter(os.Stdout) - if format == "markdown" { - table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) - table.SetCenterSeparator("|") - } - return table -} - -func printFrameHeader(title string) { - titleTable := tablewriter.NewWriter(os.Stdout) - titleTable.SetHeader([]string{title}) - titleTable.SetAutoWrapText(false) - titleTable.SetAutoFormatHeaders(true) - titleTable.SetHeaderAlignment(tablewriter.ALIGN_CENTER) - titleTable.SetRowSeparator("\n") - titleTable.SetHeaderLine(false) - titleTable.SetBorder(false) - titleTable.SetTablePadding("\t") // pad with tabs - titleTable.SetNoWhiteSpace(true) - titleTable.Render() -} - -func printHierarchicalFrame(frame data.HierarchicalFrame, format string, maxRows, maxColumns int) error { - err := printDatabaseFrame(frame, format, maxRows, maxColumns) - if err != nil { - return err - } - for _, subFrame := range frame.SubFrames { - err = printHierarchicalFrame(subFrame, format, maxRows, maxColumns) - if err != nil { - return err - } - } - return nil -} - -func printDatabaseFrame(frame data.Frame, format string, maxRows, maxColumns int) error { - table := createTable(format) - table.SetAutoWrapText(false) - columns := len(frame.Columns()) - if maxColumns > 0 && maxColumns < columns { - columns = maxColumns - } - table.SetHeader(frame.Columns()[:columns]) - r := 0 - trunColumns := 0 - for { - values, ok, err := frame.Next() - if !ok || r == maxRows { - table.Render() - if trunColumns > 0 { - warning(fmt.Sprintf("Truncated %d columns, more available...", trunColumns)) - } - if r == maxRows { - warning("Truncated rows, more available...") - } - return err - } - if err != nil { - return err - } - columns := len(values) - // -1 means unlimited - if maxColumns > 0 && maxColumns < columns { - trunColumns = columns - maxColumns - columns = maxColumns - } - row := make([]string, columns) - for i, value := range values { - if i == columns { - break - } - row[i] = fmt.Sprintf("%v", value) - } - table.Append(row) - r++ - } -} - -// currently we dump the whole config - useless in parts -func printConfigFrame(frame data.Frame, format string) error { - for { - values, ok, err := frame.Next() - if !ok { - return err - } - if err != nil { - return err - } - configFile := values[0].(data.File) - dat, err := os.ReadFile(configFile.FilePath()) - if err != nil { - return err - } - // create a table per row - as each will be a file - table := createTable(format) - table.SetAutoWrapText(false) - table.SetAutoFormatHeaders(false) - table.ClearRows() - table.SetHeader([]string{configFile.FilePath()}) - table.Append([]string{string(dat)}) - table.Render() - } -} - -func printDirectoryFileFrame(frame data.Frame, format string, maxRows int) error { - for { - values, ok, err := frame.Next() - if !ok { - - return err - } - if err != nil { - return err - } - path := values[0].(data.SimpleFile) - file, err := os.Open(path.FilePath()) - if err != nil { - // failure on one file causes rest to be ignored in frame...we could improve this - return errors.Wrapf(err, "Unable to read file %s", path.FilePath()) - } - scanner := bufio.NewScanner(file) - i := 0 - // create a table per row - as each will be a file - table := createTable(format) - table.SetAutoWrapText(false) - table.SetAutoFormatHeaders(false) - table.ClearRows() - table.SetHeader([]string{path.FilePath()}) - for scanner.Scan() { - if i == maxRows { - fmt.Println() - table.Render() - warning("Truncated lines, more available...") - fmt.Print("\n") - break - } - table.Append([]string{scanner.Text()}) - i++ - } - } -} - -// prints a warning -func warning(s string) { - fmt.Printf("\x1b[%dm%v\x1b[0m%s\n", 33, "WARNING: ", s) -} - -func (r ReportOutput) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringOptions{ - Value: "default", - Options: []string{"default", "markdown"}, - Param: config.NewParam("format", "Format of tables. Default is terminal friendly.", false), - }, - config.BoolParam{ - Value: false, - Param: config.NewParam("continue", "Print report with no interaction", false), - }, - config.IntParam{ - Value: 10, - Param: config.NewParam("row_limit", "Max Rows to print per frame.", false), - }, - config.IntParam{ - Value: 8, - Param: config.NewParam("column_limit", "Max Columns to print per frame. Negative is unlimited.", false), - }, - }, - } -} - -func (r ReportOutput) Description() string { - return "Writes out the diagnostic bundle to the terminal as a simple report." -} - -// here we register the output for use -func init() { - outputs.Register(OutputName, func() (outputs.Output, error) { - return ReportOutput{}, nil - }) -} diff --git a/programs/diagnostics/internal/platform/config/models.go b/programs/diagnostics/internal/platform/config/models.go deleted file mode 100644 index 6c76b8f149b..00000000000 --- a/programs/diagnostics/internal/platform/config/models.go +++ /dev/null @@ -1,129 +0,0 @@ -package config - -import ( - "fmt" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" -) - -type ConfigParam interface { - Name() string - Required() bool - Description() string - validate(defaultConfig ConfigParam) error -} - -type Configuration struct { - Params []ConfigParam -} - -type Param struct { - name string - description string - required bool -} - -func NewParam(name string, description string, required bool) Param { - return Param{ - name: name, - description: description, - required: required, - } -} - -func (bp Param) Name() string { - return bp.name -} - -func (bp Param) Required() bool { - return bp.required -} - -func (bp Param) Description() string { - return bp.description -} - -func (bp Param) validate(defaultConfig ConfigParam) error { - return nil -} - -func (c Configuration) GetConfigParam(paramName string) (ConfigParam, error) { - for _, param := range c.Params { - if param.Name() == paramName { - return param, nil - } - } - return nil, fmt.Errorf("%s does not exist", paramName) -} - -// ValidateConfig finds the intersection of a config c and a default config. Requires all possible params to be in default. -func (c Configuration) ValidateConfig(defaultConfig Configuration) (Configuration, error) { - var finalParams []ConfigParam - for _, defaultParam := range defaultConfig.Params { - setParam, err := c.GetConfigParam(defaultParam.Name()) - if err == nil { - // check the set value is valid - if err := setParam.validate(defaultParam); err != nil { - return Configuration{}, fmt.Errorf("parameter %s is invalid - %s", defaultParam.Name(), err.Error()) - } - finalParams = append(finalParams, setParam) - } else if defaultParam.Required() { - return Configuration{}, fmt.Errorf("missing required parameter %s - %s", defaultParam.Name(), err.Error()) - } else { - finalParams = append(finalParams, defaultParam) - } - } - return Configuration{ - Params: finalParams, - }, nil -} - -type StringParam struct { - Param - Value string - AllowEmpty bool -} - -func (sp StringParam) validate(defaultConfig ConfigParam) error { - dsp := defaultConfig.(StringParam) - if !dsp.AllowEmpty && strings.TrimSpace(sp.Value) == "" { - return fmt.Errorf("%s cannot be empty", sp.Name()) - } - // if the parameter is not required it doesn't matter - return nil -} - -type StringListParam struct { - Param - Values []string -} - -type StringOptions struct { - Param - Options []string - Value string - AllowEmpty bool -} - -func (so StringOptions) validate(defaultConfig ConfigParam) error { - dso := defaultConfig.(StringOptions) - if !dso.AllowEmpty && strings.TrimSpace(so.Value) == "" { - return fmt.Errorf("%s cannot be empty", so.Name()) - } - if !utils.Contains(dso.Options, so.Value) { - return fmt.Errorf("%s is not a valid value for %s - %v", so.Value, so.Name(), so.Options) - } - // if the parameter is not required it doesn't matter - return nil -} - -type IntParam struct { - Param - Value int64 -} - -type BoolParam struct { - Param - Value bool -} diff --git a/programs/diagnostics/internal/platform/config/models_test.go b/programs/diagnostics/internal/platform/config/models_test.go deleted file mode 100644 index 916d20ec28b..00000000000 --- a/programs/diagnostics/internal/platform/config/models_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package config_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/stretchr/testify/require" -) - -var conf = config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - Values: []string{"some", "values"}, - Param: config.NewParam("paramA", "", false), - }, - config.StringParam{ - Value: "random", - Param: config.NewParam("paramB", "", true), - }, - config.StringParam{ - Value: "", - AllowEmpty: true, - Param: config.NewParam("paramC", "", false), - }, - config.StringOptions{ - Value: "random", - Options: []string{"random", "very_random", "very_very_random"}, - Param: config.NewParam("paramD", "", false), - AllowEmpty: true, - }, - }, -} - -func TestGetConfigParam(t *testing.T) { - - t.Run("can find get config param by name", func(t *testing.T) { - paramA, err := conf.GetConfigParam("paramA") - require.Nil(t, err) - require.NotNil(t, paramA) - require.IsType(t, config.StringListParam{}, paramA) - stringListParam, ok := paramA.(config.StringListParam) - require.True(t, ok) - require.False(t, stringListParam.Required()) - require.Equal(t, stringListParam.Name(), "paramA") - require.ElementsMatch(t, stringListParam.Values, []string{"some", "values"}) - }) - - t.Run("throws error on missing element", func(t *testing.T) { - paramZ, err := conf.GetConfigParam("paramZ") - require.Nil(t, paramZ) - require.NotNil(t, err) - require.Equal(t, err.Error(), "paramZ does not exist") - }) -} - -func TestValidateConfig(t *testing.T) { - - t.Run("validate adds the default and allows override", func(t *testing.T) { - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "custom", - Param: config.NewParam("paramB", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, err) - require.NotNil(t, newConf) - require.Len(t, newConf.Params, 4) - // check first param - require.IsType(t, config.StringListParam{}, newConf.Params[0]) - stringListParam, ok := newConf.Params[0].(config.StringListParam) - require.True(t, ok) - require.False(t, stringListParam.Required()) - require.Equal(t, stringListParam.Name(), "paramA") - require.ElementsMatch(t, stringListParam.Values, []string{"some", "values"}) - // check second param - require.IsType(t, config.StringParam{}, newConf.Params[1]) - stringParam, ok := newConf.Params[1].(config.StringParam) - require.True(t, ok) - require.True(t, stringParam.Required()) - require.Equal(t, "paramB", stringParam.Name()) - require.Equal(t, "custom", stringParam.Value) - }) - - t.Run("validate errors if missing param", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - Values: []string{"some", "values"}, - Param: config.NewParam("paramA", "", false), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, newConf.Params) - require.NotNil(t, err) - require.Equal(t, "missing required parameter paramB - paramB does not exist", err.Error()) - }) - - t.Run("validate errors if invalid string value", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("paramB", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, newConf.Params) - require.NotNil(t, err) - require.Equal(t, "parameter paramB is invalid - paramB cannot be empty", err.Error()) - }) - - t.Run("allow empty string value if specified", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("paramC", "", true), - }, - config.StringParam{ - Value: "custom", - Param: config.NewParam("paramB", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.NotNil(t, newConf.Params) - require.Nil(t, err) - }) - - t.Run("validate errors if invalid string options value", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "not_random", - Param: config.NewParam("paramB", "", true), - }, - config.StringOptions{ - Value: "custom", - // this isn't ideal we need to ensure options are set for this to validate correctly - Options: []string{"random", "very_random", "very_very_random"}, - Param: config.NewParam("paramD", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, newConf.Params) - require.NotNil(t, err) - require.Equal(t, "parameter paramD is invalid - custom is not a valid value for paramD - [random very_random very_very_random]", err.Error()) - }) - - t.Run("allow empty string value for StringOptions if specified", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "custom", - Param: config.NewParam("paramB", "", true), - }, - config.StringOptions{ - Param: config.Param{}, - // this isn't ideal we need to ensure options are set for this to validate correctly - Options: []string{"random", "very_random", "very_very_random"}, - Value: "", - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.NotNil(t, newConf.Params) - require.Nil(t, err) - }) - - //TODO: Do we need to test if parameters of the same name but wrong type are passed?? -} diff --git a/programs/diagnostics/internal/platform/config/utils.go b/programs/diagnostics/internal/platform/config/utils.go deleted file mode 100644 index 5f84c38d4f4..00000000000 --- a/programs/diagnostics/internal/platform/config/utils.go +++ /dev/null @@ -1,74 +0,0 @@ -package config - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" -) - -func ReadStringListValues(conf Configuration, paramName string) ([]string, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return nil, err - } - value, ok := param.(StringListParam) - if !ok { - value, ok = param.(StringListParam) - if !ok { - return nil, fmt.Errorf("%s must be a list of strings", paramName) - } - } - - return value.Values, nil -} - -func ReadStringValue(conf Configuration, paramName string) (string, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return "", err - } - value, ok := param.(StringParam) - if !ok { - return "", fmt.Errorf("%s must be a list of strings", paramName) - } - return value.Value, nil -} - -func ReadIntValue(conf Configuration, paramName string) (int64, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return 0, err - } - value, ok := param.(IntParam) - if !ok { - return 9, fmt.Errorf("%s must be an unsigned integer", paramName) - } - return value.Value, nil -} - -func ReadBoolValue(conf Configuration, paramName string) (bool, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return false, err - } - value, ok := param.(BoolParam) - if !ok { - return false, fmt.Errorf("%s must be a boolean", paramName) - } - return value.Value, nil -} - -func ReadStringOptionsValue(conf Configuration, paramName string) (string, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return "", err - } - value, ok := param.(StringOptions) - if !ok { - return "", fmt.Errorf("%s must be a string options", paramName) - } - if !utils.Contains(value.Options, value.Value) { - return "", fmt.Errorf("%s is not a valid option in %v for the the parameter %s", value.Value, value.Options, paramName) - } - return value.Value, nil -} diff --git a/programs/diagnostics/internal/platform/config/utils_test.go b/programs/diagnostics/internal/platform/config/utils_test.go deleted file mode 100644 index 9e03e5e69d2..00000000000 --- a/programs/diagnostics/internal/platform/config/utils_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package config_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/stretchr/testify/require" -) - -func TestReadStringListValues(t *testing.T) { - - t.Run("can find a string list param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringListParam{ - Values: []string{"licenses", "settings"}, - Param: config.NewParam("exclude_tables", "Specify list of tables not to collect", false), - }, - }, - } - excludeTables, err := config.ReadStringListValues(conf, "exclude_tables") - require.Nil(t, err) - require.Equal(t, []string{"licenses", "settings"}, excludeTables) - }) - -} - -func TestReadStringValue(t *testing.T) { - - t.Run("can find a string param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringParam{ - Value: "/tmp/dump", - Param: config.NewParam("directory", "Specify a directory", false), - }, - }, - } - directory, err := config.ReadStringValue(conf, "directory") - require.Nil(t, err) - require.Equal(t, "/tmp/dump", directory) - }) - -} - -func TestReadIntValue(t *testing.T) { - t.Run("can find an integer param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - // nil means include everything - Value: 10000, - Param: config.NewParam("row_limit", "Max Rows to collect", false), - }, - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringParam{ - Value: "/tmp/dump", - Param: config.NewParam("directory", "Specify a directory", false), - }, - }, - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - require.Nil(t, err) - require.Equal(t, int64(10000), rowLimit) - }) - -} - -func TestReadBoolValue(t *testing.T) { - t.Run("can find a boolean param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.BoolParam{ - // nil means include everything - Value: true, - Param: config.NewParam("compress", "Compress data", false), - }, - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringParam{ - Value: "/tmp/dump", - Param: config.NewParam("directory", "Specify a directory", false), - }, - }, - } - - compress, err := config.ReadBoolValue(conf, "compress") - require.Nil(t, err) - require.True(t, compress) - }) -} - -func TestReadStringOptionsValue(t *testing.T) { - t.Run("can find a string value in a list of options", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringOptions{ - Param: config.NewParam("format", "List of formats", false), - Options: []string{"csv", "tsv", "binary", "json", "ndjson"}, - Value: "csv", - AllowEmpty: false, - }, - }, - } - format, err := config.ReadStringOptionsValue(conf, "format") - require.Nil(t, err) - require.Equal(t, "csv", format) - }) - - t.Run("errors on invalid value", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringOptions{ - Param: config.NewParam("format", "List of formats", false), - Options: []string{"csv", "tsv", "binary", "json", "ndjson"}, - Value: "random", - AllowEmpty: false, - }, - }, - } - format, err := config.ReadStringOptionsValue(conf, "format") - require.Equal(t, "random is not a valid option in [csv tsv binary json ndjson] for the the parameter format", err.Error()) - require.Equal(t, "", format) - }) -} diff --git a/programs/diagnostics/internal/platform/data/bundle.go b/programs/diagnostics/internal/platform/data/bundle.go deleted file mode 100644 index e4eeede659e..00000000000 --- a/programs/diagnostics/internal/platform/data/bundle.go +++ /dev/null @@ -1,27 +0,0 @@ -package data - -import ( - "strings" -) - -// DiagnosticBundle contains the results from a Collector -// each frame can represent a table or collection of data files. By allowing multiple frames a single DiagnosticBundle -// can potentially contain many related tables -type DiagnosticBundle struct { - Frames map[string]Frame - // Errors is a property to be set if the Collector has an error. This can be used to indicate a partial collection - // and failed frames - Errors FrameErrors -} - -type FrameErrors struct { - Errors []error -} - -func (fe *FrameErrors) Error() string { - errors := make([]string, len(fe.Errors)) - for i := range errors { - errors[i] = fe.Errors[i].Error() - } - return strings.Join(errors, "\n") -} diff --git a/programs/diagnostics/internal/platform/data/bundle_test.go b/programs/diagnostics/internal/platform/data/bundle_test.go deleted file mode 100644 index ff9cfc2cf56..00000000000 --- a/programs/diagnostics/internal/platform/data/bundle_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package data_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/stretchr/testify/require" -) - -func TestBundleError(t *testing.T) { - - t.Run("can get a bundle error", func(t *testing.T) { - errs := make([]error, 3) - errs[0] = errors.New("Error 1") - errs[1] = errors.New("Error 2") - errs[2] = errors.New("Error 3") - fErrors := data.FrameErrors{ - Errors: errs, - } - require.Equal(t, `Error 1 -Error 2 -Error 3`, fErrors.Error()) - - }) -} diff --git a/programs/diagnostics/internal/platform/data/database.go b/programs/diagnostics/internal/platform/data/database.go deleted file mode 100644 index d49317d8719..00000000000 --- a/programs/diagnostics/internal/platform/data/database.go +++ /dev/null @@ -1,88 +0,0 @@ -package data - -import ( - "database/sql" - "fmt" - "reflect" - "strings" -) - -type DatabaseFrame struct { - name string - ColumnNames []string - rows *sql.Rows - columnTypes []*sql.ColumnType - vars []interface{} -} - -func NewDatabaseFrame(name string, rows *sql.Rows) (DatabaseFrame, error) { - databaseFrame := DatabaseFrame{} - columnTypes, err := rows.ColumnTypes() - if err != nil { - return DatabaseFrame{}, err - } - databaseFrame.columnTypes = columnTypes - databaseFrame.name = name - vars := make([]interface{}, len(columnTypes)) - columnNames := make([]string, len(columnTypes)) - for i := range columnTypes { - value := reflect.Zero(columnTypes[i].ScanType()).Interface() - vars[i] = &value - columnNames[i] = columnTypes[i].Name() - } - databaseFrame.ColumnNames = columnNames - databaseFrame.vars = vars - databaseFrame.rows = rows - return databaseFrame, nil -} - -func (f DatabaseFrame) Next() ([]interface{}, bool, error) { - values := make([]interface{}, len(f.columnTypes)) - for f.rows.Next() { - if err := f.rows.Scan(f.vars...); err != nil { - return nil, false, err - } - for i := range f.columnTypes { - ptr := reflect.ValueOf(f.vars[i]) - values[i] = ptr.Elem().Interface() - } - return values, true, nil //nolint - } - // TODO: raise issue as this seems to always raise an error - //err := f.rows.Err() - f.rows.Close() - return nil, false, nil -} - -func (f DatabaseFrame) Columns() []string { - return f.ColumnNames -} - -func (f DatabaseFrame) Name() string { - return f.name -} - -type Order int - -const ( - Asc Order = 1 - Desc Order = 2 -) - -type OrderBy struct { - Column string - Order Order -} - -func (o OrderBy) String() string { - if strings.TrimSpace(o.Column) == "" { - return "" - } - switch o.Order { - case Asc: - return fmt.Sprintf(" ORDER BY %s ASC", o.Column) - case Desc: - return fmt.Sprintf(" ORDER BY %s DESC", o.Column) - } - return "" -} diff --git a/programs/diagnostics/internal/platform/data/database_test.go b/programs/diagnostics/internal/platform/data/database_test.go deleted file mode 100644 index 57d89e78efc..00000000000 --- a/programs/diagnostics/internal/platform/data/database_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package data_test - -import ( - "database/sql" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/DATA-DOG/go-sqlmock" - "github.com/stretchr/testify/require" -) - -func TestString(t *testing.T) { - t.Run("can order by asc", func(t *testing.T) { - orderBy := data.OrderBy{ - Column: "created_at", - Order: data.Asc, - } - require.Equal(t, " ORDER BY created_at ASC", orderBy.String()) - }) - - t.Run("can order by desc", func(t *testing.T) { - orderBy := data.OrderBy{ - Column: "created_at", - Order: data.Desc, - } - require.Equal(t, " ORDER BY created_at DESC", orderBy.String()) - }) - -} - -func TestNextDatabaseFrame(t *testing.T) { - - t.Run("can iterate sql rows", func(t *testing.T) { - rowValues := [][]interface{}{ - {int64(1), "post_1", "hello"}, - {int64(2), "post_2", "world"}, - {int64(3), "post_3", "goodbye"}, - {int64(4), "post_4", "world"}, - } - mockRows := sqlmock.NewRows([]string{"id", "title", "body"}) - for i := range rowValues { - mockRows.AddRow(rowValues[i][0], rowValues[i][1], rowValues[i][2]) - } - rows := mockRowsToSqlRows(mockRows) - dbFrame, err := data.NewDatabaseFrame("test", rows) - require.ElementsMatch(t, dbFrame.Columns(), []string{"id", "title", "body"}) - require.Nil(t, err) - i := 0 - for { - values, ok, err := dbFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 3) - require.ElementsMatch(t, values, rowValues[i]) - i++ - } - require.Equal(t, 4, i) - }) - - t.Run("can iterate empty sql rows", func(t *testing.T) { - mockRows := sqlmock.NewRows([]string{"id", "title", "body"}) - rows := mockRowsToSqlRows(mockRows) - dbFrame, err := data.NewDatabaseFrame("test", rows) - require.ElementsMatch(t, dbFrame.Columns(), []string{"id", "title", "body"}) - require.Nil(t, err) - i := 0 - for { - _, ok, err := dbFrame.Next() - require.Nil(t, err) - if !ok { - break - } - i++ - } - require.Equal(t, 0, i) - }) -} - -func mockRowsToSqlRows(mockRows *sqlmock.Rows) *sql.Rows { - db, mock, _ := sqlmock.New() - mock.ExpectQuery("select").WillReturnRows(mockRows) - rows, _ := db.Query("select") - return rows -} diff --git a/programs/diagnostics/internal/platform/data/field.go b/programs/diagnostics/internal/platform/data/field.go deleted file mode 100644 index 5e80fc1f467..00000000000 --- a/programs/diagnostics/internal/platform/data/field.go +++ /dev/null @@ -1,8 +0,0 @@ -package data - -type Field struct { - // Name of the field - Name string - // A list of fields that must implement FieldType interface - Values []interface{} -} diff --git a/programs/diagnostics/internal/platform/data/file.go b/programs/diagnostics/internal/platform/data/file.go deleted file mode 100644 index 9760b4b6906..00000000000 --- a/programs/diagnostics/internal/platform/data/file.go +++ /dev/null @@ -1,444 +0,0 @@ -package data - -import ( - "bufio" - "encoding/xml" - "io/ioutil" - "os" - "path" - "path/filepath" - "regexp" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" -) - -type File interface { - Copy(destPath string, removeSensitive bool) error - FilePath() string -} - -type SimpleFile struct { - Path string -} - -// Copy supports removeSensitive for other file types but for a simple file this doesn't do anything -func (s SimpleFile) Copy(destPath string, removeSensitive bool) error { - // simple copy easiest - if err := utils.CopyFile(s.FilePath(), destPath); err != nil { - return errors.Wrapf(err, "unable to copy file %s", s.FilePath()) - } - return nil -} - -func (s SimpleFile) FilePath() string { - return s.Path -} - -func NewFileFrame(name string, filePaths []string) FileFrame { - i := 0 - files := make([]File, len(filePaths)) - for i, path := range filePaths { - files[i] = SimpleFile{ - Path: path, - } - } - return FileFrame{ - name: name, - i: &i, - files: files, - } -} - -type FileFrame struct { - name string - i *int - files []File -} - -func (f FileFrame) Next() ([]interface{}, bool, error) { - if len(f.files) == *(f.i) { - return nil, false, nil - } - file := f.files[*f.i] - *f.i++ - value := make([]interface{}, 1) - value[0] = file - return value, true, nil -} - -func (f FileFrame) Columns() []string { - return []string{"files"} -} - -func (f FileFrame) Name() string { - return f.name -} - -// DirectoryFileFrame represents a set of files under a directory -type DirectoryFileFrame struct { - FileFrame - Directory string -} - -func NewFileDirectoryFrame(directory string, exts []string) (DirectoryFileFrame, []error) { - filePaths, errs := utils.ListFilesInDirectory(directory, exts) - files := make([]File, len(filePaths)) - for i, path := range filePaths { - files[i] = SimpleFile{ - Path: path, - } - } - i := 0 - return DirectoryFileFrame{ - Directory: directory, - FileFrame: FileFrame{ - files: files, - i: &i, - }, - }, errs -} - -func (f DirectoryFileFrame) Next() ([]interface{}, bool, error) { - if len(f.files) == *(f.i) { - return nil, false, nil - } - file := f.files[*f.i] - *f.i++ - value := make([]interface{}, 1) - value[0] = file - return value, true, nil -} - -func (f DirectoryFileFrame) Columns() []string { - return []string{"files"} -} - -func (f DirectoryFileFrame) Name() string { - return f.Directory -} - -type ConfigFile interface { - File - FindLogPaths() ([]string, error) - FindIncludedConfig() (ConfigFile, error) - IsIncluded() bool -} - -type ConfigFileFrame struct { - i *int - Directory string - files []ConfigFile -} - -func (f ConfigFileFrame) Next() ([]interface{}, bool, error) { - if len(f.files) == *(f.i) { - return nil, false, nil - } - file := f.files[*f.i] - *f.i++ - value := make([]interface{}, 1) - value[0] = file - return value, true, nil -} - -func (f ConfigFileFrame) Name() string { - return f.Directory -} - -func NewConfigFileFrame(directory string) (ConfigFileFrame, []error) { - files, errs := utils.ListFilesInDirectory(directory, []string{"*.xml", "*.yaml", "*.yml"}) - // we can't predict the length because of include files - var configs []ConfigFile - - for _, path := range files { - var configFile ConfigFile - switch ext := filepath.Ext(path); ext { - case ".xml": - configFile = XmlConfigFile{ - Path: path, - Included: false, - } - case ".yml": - configFile = YamlConfigFile{ - Path: path, - Included: false, - } - case ".yaml": - configFile = YamlConfigFile{ - Path: path, - } - } - if configFile != nil { - configs = append(configs, configFile) - // add any included configs - iConf, err := configFile.FindIncludedConfig() - if err != nil { - errs = append(errs, err) - } else { - if iConf.FilePath() != "" { - configs = append(configs, iConf) - } - } - } - } - i := 0 - - return ConfigFileFrame{ - i: &i, - Directory: directory, - files: configs, - }, errs -} - -func (f ConfigFileFrame) Columns() []string { - return []string{"config"} -} - -func (f ConfigFileFrame) FindLogPaths() (logPaths []string, errors []error) { - for _, configFile := range f.files { - paths, err := configFile.FindLogPaths() - if err != nil { - errors = append(errors, err) - } else { - logPaths = append(logPaths, paths...) - } - } - return logPaths, errors -} - -type XmlConfigFile struct { - Path string - Included bool -} - -// these patterns will be used to remove sensitive content - matches of the pattern will be replaced with the key -var xmlSensitivePatterns = map[string]*regexp.Regexp{ - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), -} - -func (x XmlConfigFile) Copy(destPath string, removeSensitive bool) error { - if !removeSensitive { - // simple copy easiest - if err := utils.CopyFile(x.FilePath(), destPath); err != nil { - return errors.Wrapf(err, "unable to copy file %s", x.FilePath()) - } - return nil - } - return sensitiveFileCopy(x.FilePath(), destPath, xmlSensitivePatterns) -} - -func (x XmlConfigFile) FilePath() string { - return x.Path -} - -func (x XmlConfigFile) IsIncluded() bool { - return x.Included -} - -type XmlLoggerConfig struct { - XMLName xml.Name `xml:"logger"` - ErrorLog string `xml:"errorlog"` - Log string `xml:"log"` -} - -type YandexXMLConfig struct { - XMLName xml.Name `xml:"yandex"` - Clickhouse XmlLoggerConfig `xml:"logger"` - IncludeFrom string `xml:"include_from"` -} - -type XmlConfig struct { - XMLName xml.Name `xml:"clickhouse"` - Clickhouse XmlLoggerConfig `xml:"logger"` - IncludeFrom string `xml:"include_from"` -} - -func (x XmlConfigFile) UnmarshallConfig() (XmlConfig, error) { - inputFile, err := ioutil.ReadFile(x.Path) - - if err != nil { - return XmlConfig{}, err - } - var cConfig XmlConfig - err = xml.Unmarshal(inputFile, &cConfig) - if err == nil { - return XmlConfig{ - Clickhouse: cConfig.Clickhouse, - IncludeFrom: cConfig.IncludeFrom, - }, nil - } - // attempt to marshall as yandex file - var yConfig YandexXMLConfig - err = xml.Unmarshal(inputFile, &yConfig) - if err != nil { - return XmlConfig{}, err - } - return XmlConfig{ - Clickhouse: yConfig.Clickhouse, - IncludeFrom: yConfig.IncludeFrom, - }, nil -} - -func (x XmlConfigFile) FindLogPaths() ([]string, error) { - var paths []string - config, err := x.UnmarshallConfig() - if err != nil { - return nil, err - } - if config.Clickhouse.Log != "" { - paths = append(paths, config.Clickhouse.Log) - } - if config.Clickhouse.ErrorLog != "" { - paths = append(paths, config.Clickhouse.ErrorLog) - } - - return paths, nil -} - -func (x XmlConfigFile) FindIncludedConfig() (ConfigFile, error) { - if x.Included { - //can't recurse - return XmlConfigFile{}, nil - } - config, err := x.UnmarshallConfig() - if err != nil { - return XmlConfigFile{}, err - } - // we need to convert this - if config.IncludeFrom != "" { - if filepath.IsAbs(config.IncludeFrom) { - return XmlConfigFile{Path: config.IncludeFrom, Included: true}, nil - } - confDir := filepath.Dir(x.FilePath()) - return XmlConfigFile{Path: path.Join(confDir, config.IncludeFrom), Included: true}, nil - } - return XmlConfigFile{}, nil -} - -type YamlConfigFile struct { - Path string - Included bool -} - -var ymlSensitivePatterns = map[string]*regexp.Regexp{ - "password: 'Replaced'": regexp.MustCompile(`password:\s*.*$`), - "password_sha256_hex: 'Replaced'": regexp.MustCompile(`password_sha256_hex:\s*.*$`), - "access_key_id: 'Replaced'": regexp.MustCompile(`access_key_id:\s*.*$`), - "secret_access_key: 'Replaced'": regexp.MustCompile(`secret_access_key:\s*.*$`), - "secret: 'Replaced'": regexp.MustCompile(`secret:\s*.*$`), -} - -func (y YamlConfigFile) Copy(destPath string, removeSensitive bool) error { - if !removeSensitive { - // simple copy easiest - if err := utils.CopyFile(y.FilePath(), destPath); err != nil { - return errors.Wrapf(err, "unable to copy file %s", y.FilePath()) - } - return nil - } - return sensitiveFileCopy(y.FilePath(), destPath, ymlSensitivePatterns) -} - -func (y YamlConfigFile) FilePath() string { - return y.Path -} - -func (y YamlConfigFile) IsIncluded() bool { - return y.Included -} - -type YamlLoggerConfig struct { - Log string - ErrorLog string -} - -type YamlConfig struct { - Logger YamlLoggerConfig - Include_From string -} - -func (y YamlConfigFile) FindLogPaths() ([]string, error) { - var paths []string - inputFile, err := ioutil.ReadFile(y.Path) - if err != nil { - return nil, err - } - var config YamlConfig - err = yaml.Unmarshal(inputFile, &config) - if err != nil { - return nil, err - } - if config.Logger.Log != "" { - paths = append(paths, config.Logger.Log) - } - if config.Logger.ErrorLog != "" { - paths = append(paths, config.Logger.ErrorLog) - } - return paths, nil -} - -func (y YamlConfigFile) FindIncludedConfig() (ConfigFile, error) { - if y.Included { - //can't recurse - return YamlConfigFile{}, nil - } - inputFile, err := ioutil.ReadFile(y.Path) - if err != nil { - return YamlConfigFile{}, err - } - var config YamlConfig - err = yaml.Unmarshal(inputFile, &config) - if err != nil { - return YamlConfigFile{}, err - } - if config.Include_From != "" { - if filepath.IsAbs(config.Include_From) { - return YamlConfigFile{Path: config.Include_From, Included: true}, nil - } - confDir := filepath.Dir(y.FilePath()) - return YamlConfigFile{Path: path.Join(confDir, config.Include_From), Included: true}, nil - } - return YamlConfigFile{}, nil -} - -func sensitiveFileCopy(sourcePath string, destPath string, patterns map[string]*regexp.Regexp) error { - destDir := filepath.Dir(destPath) - if err := os.MkdirAll(destDir, os.ModePerm); err != nil { - return errors.Wrapf(err, "unable to create directory %s", destDir) - } - // currently, we don't unmarshall into a struct - we want to preserve structure and comments. Possibly could - // be handled but for simplicity we do a line parse for now - inputFile, err := os.Open(sourcePath) - - if err != nil { - return err - } - defer inputFile.Close() - outputFile, err := os.Create(destPath) - - if err != nil { - return err - } - defer outputFile.Close() - writer := bufio.NewWriter(outputFile) - scanner := bufio.NewScanner(inputFile) - - for scanner.Scan() { - line := scanner.Text() - for repl, pattern := range patterns { - line = pattern.ReplaceAllString(line, repl) - } - _, err = writer.WriteString(line + "\n") - if err != nil { - return err - } - } - writer.Flush() - return nil -} diff --git a/programs/diagnostics/internal/platform/data/file_test.go b/programs/diagnostics/internal/platform/data/file_test.go deleted file mode 100644 index 9e305b1a5da..00000000000 --- a/programs/diagnostics/internal/platform/data/file_test.go +++ /dev/null @@ -1,263 +0,0 @@ -package data_test - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestNextFileDirectoryFrame(t *testing.T) { - t.Run("can iterate file frame", func(t *testing.T) { - tempDir := t.TempDir() - files := make([]string, 5) - for i := 0; i < 5; i++ { - fileDir := path.Join(tempDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.txt", i)) - files[i] = filepath - _, err = os.Create(filepath) - require.Nil(t, err) - } - fileFrame, errs := data.NewFileDirectoryFrame(tempDir, []string{"*.txt"}) - require.Empty(t, errs) - i := 0 - for { - values, ok, err := fileFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - require.Equal(t, files[i], values[0].(data.SimpleFile).Path) - i += 1 - } - require.Equal(t, 5, i) - }) - - t.Run("can iterate file frame when empty", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - fileFrame, errs := data.NewFileDirectoryFrame(tempDir, []string{"*"}) - require.Empty(t, errs) - i := 0 - for { - _, ok, err := fileFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) -} - -func TestNewConfigFileFrame(t *testing.T) { - t.Run("can iterate config file frame", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - i := 0 - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - filePath := values[0].(data.XmlConfigFile).FilePath() - require.True(t, strings.Contains(filePath, ".xml")) - i += 1 - } - // 5 not 3 due to the includes - require.Equal(t, 5, i) - }) - - t.Run("can iterate file frame when empty", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - configFrame, errs := data.NewConfigFileFrame(tempDir) - require.Empty(t, errs) - i := 0 - for { - _, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) -} - -func TestConfigFileFrameCopy(t *testing.T) { - t.Run("can copy non-sensitive xml config files", func(t *testing.T) { - tmrDir := t.TempDir() - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Nil(t, err) - require.True(t, ok) - configFile := values[0].(data.XmlConfigFile) - newPath := path.Join(tmrDir, filepath.Base(configFile.FilePath())) - err = configFile.Copy(newPath, false) - require.FileExists(t, newPath) - sourceInfo, _ := os.Stat(configFile.FilePath()) - destInfo, _ := os.Stat(newPath) - require.Equal(t, sourceInfo.Size(), destInfo.Size()) - require.Nil(t, err) - } - }) - - t.Run("can copy sensitive xml config files", func(t *testing.T) { - tmrDir := t.TempDir() - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - i := 0 - var checkedFiles []string - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Nil(t, err) - require.True(t, ok) - configFile := values[0].(data.XmlConfigFile) - fileName := filepath.Base(configFile.FilePath()) - newPath := path.Join(tmrDir, fileName) - err = configFile.Copy(newPath, true) - require.FileExists(t, newPath) - require.Nil(t, err) - bytes, err := ioutil.ReadFile(newPath) - require.Nil(t, err) - s := string(bytes) - checkedFiles = append(checkedFiles, fileName) - if fileName == "users.xml" || fileName == "default-password.xml" || fileName == "user-include.xml" { - require.True(t, strings.Contains(s, "Replaced") || - strings.Contains(s, "Replaced")) - require.NotContains(t, s, "REPLACE_ME") - require.NotContains(t, s, "REPLACE_ME") - } else if fileName == "config.xml" { - require.True(t, strings.Contains(s, "Replaced")) - require.True(t, strings.Contains(s, "Replaced")) - require.True(t, strings.Contains(s, "Replaced")) - require.NotContains(t, s, "REPLACE_ME") - require.NotContains(t, s, "REPLACE_ME") - require.NotContains(t, s, "REPLACE_ME") - } - i++ - } - require.ElementsMatch(t, []string{"users.xml", "default-password.xml", "user-include.xml", "config.xml", "server-include.xml"}, checkedFiles) - require.Equal(t, 5, i) - }) - - t.Run("can copy sensitive yaml config files", func(t *testing.T) { - tmrDir := t.TempDir() - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yaml")) - require.Empty(t, errs) - i := 0 - var checkedFiles []string - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Nil(t, err) - require.True(t, ok) - configFile := values[0].(data.YamlConfigFile) - fileName := filepath.Base(configFile.FilePath()) - newPath := path.Join(tmrDir, fileName) - err = configFile.Copy(newPath, true) - require.FileExists(t, newPath) - require.Nil(t, err) - bytes, err := ioutil.ReadFile(newPath) - require.Nil(t, err) - s := string(bytes) - checkedFiles = append(checkedFiles, fileName) - if fileName == "users.yaml" || fileName == "default-password.yaml" || fileName == "user-include.yaml" { - require.True(t, strings.Contains(s, "password: 'Replaced'") || - strings.Contains(s, "password_sha256_hex: 'Replaced'")) - require.NotContains(t, s, "password: 'REPLACE_ME'") - require.NotContains(t, s, "password_sha256_hex: \"REPLACE_ME\"") - } else if fileName == "config.yaml" { - require.True(t, strings.Contains(s, "access_key_id: 'Replaced'")) - require.True(t, strings.Contains(s, "secret_access_key: 'Replaced'")) - require.True(t, strings.Contains(s, "secret: 'Replaced'")) - require.NotContains(t, s, "access_key_id: 'REPLACE_ME'") - require.NotContains(t, s, "secret_access_key: REPLACE_ME") - require.NotContains(t, s, "secret: REPLACE_ME") - } - i++ - } - require.ElementsMatch(t, []string{"users.yaml", "default-password.yaml", "user-include.yaml", "config.yaml", "server-include.yaml"}, checkedFiles) - require.Equal(t, 5, i) - }) -} - -func TestConfigFileFrameFindLogPaths(t *testing.T) { - t.Run("can find xml log paths", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.ElementsMatch(t, []string{"/var/log/clickhouse-server/clickhouse-server.log", - "/var/log/clickhouse-server/clickhouse-server.err.log"}, paths) - }) - - t.Run("can handle empty log paths", func(t *testing.T) { - configFrame, errs := data.NewConfigFileFrame(t.TempDir()) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.Empty(t, paths) - }) - - t.Run("can find yaml log paths", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yaml")) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.ElementsMatch(t, []string{"/var/log/clickhouse-server/clickhouse-server.log", - "/var/log/clickhouse-server/clickhouse-server.err.log"}, paths) - }) -} - -// test the legacy format for ClickHouse xml config files with a yandex root tag -func TestYandexConfigFile(t *testing.T) { - t.Run("can find xml log paths with yandex root", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yandex_xml")) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.ElementsMatch(t, []string{"/var/log/clickhouse-server/clickhouse-server.log", - "/var/log/clickhouse-server/clickhouse-server.err.log"}, paths) - }) -} diff --git a/programs/diagnostics/internal/platform/data/frame.go b/programs/diagnostics/internal/platform/data/frame.go deleted file mode 100644 index 65978430109..00000000000 --- a/programs/diagnostics/internal/platform/data/frame.go +++ /dev/null @@ -1,11 +0,0 @@ -package data - -type BaseFrame struct { - Name string -} - -type Frame interface { - Next() ([]interface{}, bool, error) - Columns() []string - Name() string -} diff --git a/programs/diagnostics/internal/platform/data/memory.go b/programs/diagnostics/internal/platform/data/memory.go deleted file mode 100644 index 25da25cf251..00000000000 --- a/programs/diagnostics/internal/platform/data/memory.go +++ /dev/null @@ -1,35 +0,0 @@ -package data - -type MemoryFrame struct { - i *int - ColumnNames []string - Rows [][]interface{} - name string -} - -func NewMemoryFrame(name string, columns []string, rows [][]interface{}) MemoryFrame { - i := 0 - return MemoryFrame{ - i: &i, - Rows: rows, - ColumnNames: columns, - name: name, - } -} - -func (f MemoryFrame) Next() ([]interface{}, bool, error) { - if f.Rows == nil || len(f.Rows) == *(f.i) { - return nil, false, nil - } - value := f.Rows[*f.i] - *f.i++ - return value, true, nil -} - -func (f MemoryFrame) Columns() []string { - return f.ColumnNames -} - -func (f MemoryFrame) Name() string { - return f.name -} diff --git a/programs/diagnostics/internal/platform/data/memory_test.go b/programs/diagnostics/internal/platform/data/memory_test.go deleted file mode 100644 index fcc02e37d32..00000000000 --- a/programs/diagnostics/internal/platform/data/memory_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package data_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestNextMemoryFrame(t *testing.T) { - t.Run("can iterate memory frame", func(t *testing.T) { - columns := []string{"Filesystem", "Size", "Used", "Avail", "Use%", "Mounted on"} - rows := [][]interface{}{ - {"sysfs", 0, 0, 0, 0, "/sys"}, - {"proc", 0, 0, 0, 0, "/proc"}, - {"udev", 33357840384, 0, 33357840384, 0, "/dev"}, - {"devpts", 0, 0, 0, 0, "/dev/pts"}, - {"tmpfs", 6682607616, 2228224, 6680379392, 1, "/run"}, - {"/dev/mapper/system-root", 1938213220352, 118136926208, 1721548947456, 7.000000000000001, "/"}, - } - memoryFrame := data.NewMemoryFrame("disks", columns, rows) - i := 0 - for { - values, ok, err := memoryFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.ElementsMatch(t, values, rows[i]) - require.Len(t, values, 6) - i += 1 - } - require.Equal(t, 6, i) - }) - - t.Run("can iterate memory frame when empty", func(t *testing.T) { - memoryFrame := data.NewMemoryFrame("test", []string{}, [][]interface{}{}) - i := 0 - for { - _, ok, err := memoryFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) - - t.Run("can iterate memory frame when empty", func(t *testing.T) { - memoryFrame := data.MemoryFrame{} - i := 0 - for { - _, ok, err := memoryFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) -} diff --git a/programs/diagnostics/internal/platform/data/misc.go b/programs/diagnostics/internal/platform/data/misc.go deleted file mode 100644 index a03213c4f46..00000000000 --- a/programs/diagnostics/internal/platform/data/misc.go +++ /dev/null @@ -1,27 +0,0 @@ -package data - -func NewHierarchicalFrame(name string, frame Frame, subFrames []HierarchicalFrame) HierarchicalFrame { - return HierarchicalFrame{ - name: name, - DataFrame: frame, - SubFrames: subFrames, - } -} - -type HierarchicalFrame struct { - name string - DataFrame Frame - SubFrames []HierarchicalFrame -} - -func (hf HierarchicalFrame) Name() string { - return hf.name -} - -func (hf HierarchicalFrame) Columns() []string { - return hf.DataFrame.Columns() -} - -func (hf HierarchicalFrame) Next() ([]interface{}, bool, error) { - return hf.DataFrame.Next() -} diff --git a/programs/diagnostics/internal/platform/database/native.go b/programs/diagnostics/internal/platform/database/native.go deleted file mode 100644 index 45b9af0349e..00000000000 --- a/programs/diagnostics/internal/platform/database/native.go +++ /dev/null @@ -1,95 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "net/url" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - _ "github.com/ClickHouse/clickhouse-go/v2" - "github.com/pkg/errors" -) - -type ClickhouseNativeClient struct { - host string - connection *sql.DB -} - -func NewNativeClient(host string, port uint16, username string, password string) (*ClickhouseNativeClient, error) { - // debug output ?debug=true - connection, err := sql.Open("clickhouse", fmt.Sprintf("clickhouse://%s:%s@%s:%d/", url.QueryEscape(username), url.QueryEscape(password), host, port)) - if err != nil { - return &ClickhouseNativeClient{}, err - } - if err := connection.Ping(); err != nil { - return &ClickhouseNativeClient{}, err - } - return &ClickhouseNativeClient{ - host: host, - connection: connection, - }, nil -} - -func (c *ClickhouseNativeClient) Ping() error { - return c.connection.Ping() -} - -func (c *ClickhouseNativeClient) ReadTable(databaseName string, tableName string, excludeColumns []string, orderBy data.OrderBy, limit int64) (data.Frame, error) { - exceptClause := "" - if len(excludeColumns) > 0 { - exceptClause = fmt.Sprintf("EXCEPT(%s) ", strings.Join(excludeColumns, ",")) - } - limitClause := "" - if limit >= 0 { - limitClause = fmt.Sprintf(" LIMIT %d", limit) - } - rows, err := c.connection.Query(fmt.Sprintf("SELECT * %sFROM %s.%s%s%s", exceptClause, databaseName, tableName, orderBy.String(), limitClause)) - if err != nil { - return data.DatabaseFrame{}, err - } - return data.NewDatabaseFrame(fmt.Sprintf("%s.%s", databaseName, tableName), rows) -} - -func (c *ClickhouseNativeClient) ReadTableNamesForDatabase(databaseName string) ([]string, error) { - rows, err := c.connection.Query(fmt.Sprintf("SHOW TABLES FROM %s", databaseName)) - if err != nil { - return nil, err - } - defer rows.Close() - var tableNames []string - var name string - for rows.Next() { - if err := rows.Scan(&name); err != nil { - return nil, err - } - tableNames = append(tableNames, name) - } - return tableNames, nil -} - -func (c *ClickhouseNativeClient) ExecuteStatement(id string, statement string) (data.Frame, error) { - rows, err := c.connection.Query(statement) - if err != nil { - return data.DatabaseFrame{}, err - } - return data.NewDatabaseFrame(id, rows) -} - -func (c *ClickhouseNativeClient) Version() (string, error) { - frame, err := c.ExecuteStatement("version", "SELECT version() as version") - if err != nil { - return "", err - } - values, ok, err := frame.Next() - if err != nil { - return "", err - } - if !ok { - return "", errors.New("unable to read ClickHouse version") - } - if len(values) != 1 { - return "", errors.New("unable to read ClickHouse version - no rows returned") - } - return values[0].(string), nil -} diff --git a/programs/diagnostics/internal/platform/database/native_test.go b/programs/diagnostics/internal/platform/database/native_test.go deleted file mode 100644 index 7028a4b4800..00000000000 --- a/programs/diagnostics/internal/platform/database/native_test.go +++ /dev/null @@ -1,289 +0,0 @@ -//go:build !no_docker - -package database_test - -import ( - "context" - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/docker/go-connections/nat" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -func createClickHouseContainer(t *testing.T, ctx context.Context) (testcontainers.Container, nat.Port) { - // create a ClickHouse container - cwd, err := os.Getwd() - if err != nil { - // can't test without current directory - panic(err) - } - - // for now, we test against a hardcoded database-server version but we should make this a property - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../../testdata/docker/admin.xml"), - }, - Target: "/etc/clickhouse-server/users.d/admin.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, _ := clickhouseContainer.MappedPort(ctx, "9000") - if err != nil { - // can't test without container's port - panic(err) - } - - t.Setenv("CLICKHOUSE_DB_PORT", p.Port()) - - return clickhouseContainer, p -} - -func getClient(t *testing.T, mappedPort int) *database.ClickhouseNativeClient { - clickhouseClient, err := database.NewNativeClient("localhost", uint16(mappedPort), "", "") - if err != nil { - t.Fatalf("unable to build client : %v", err) - } - return clickhouseClient -} - -func TestReadTableNamesForDatabase(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - t.Run("client can read tables for a database", func(t *testing.T) { - tables, err := clickhouseClient.ReadTableNamesForDatabase("system") - require.Nil(t, err) - require.GreaterOrEqual(t, len(tables), 70) - require.Contains(t, tables, "merge_tree_settings") - }) -} - -func TestReadTable(t *testing.T) { - t.Run("client can get all rows for system.disks table", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - // we read the table system.disks as this should contain only 1 row - frame, err := clickhouseClient.ReadTable("system", "disks", []string{}, data.OrderBy{}, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [9]string{"name", "path", "free_space", "total_space", "unreserved_space", "keep_free_space", "type", "is_encrypted", "cache_path"}) - i := 0 - for { - values, ok, err := frame.Next() - if i == 0 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, "default", values[0]) - require.Equal(t, "/var/lib/clickhouse/", values[1]) - require.Greater(t, values[2], uint64(0)) - require.Greater(t, values[3], uint64(0)) - require.Greater(t, values[4], uint64(0)) - require.Equal(t, values[5], uint64(0)) - require.Equal(t, "local", values[6]) - require.Equal(t, values[7], uint8(0)) - require.Equal(t, values[8], "") - } else { - require.False(t, ok) - break - } - i += 1 - } - }) - - t.Run("client can get all rows for system.databases table", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - // we read the table system.databases as this should be small and consistent on fresh db instances - frame, err := clickhouseClient.ReadTable("system", "databases", []string{}, data.OrderBy{}, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [6]string{"name", "engine", "data_path", "metadata_path", "uuid", "comment"}) - expectedRows := [4][3]string{{"INFORMATION_SCHEMA", "Memory", "/var/lib/clickhouse/"}, - {"default", "Atomic", "/var/lib/clickhouse/store/"}, - {"information_schema", "Memory", "/var/lib/clickhouse/"}, - {"system", "Atomic", "/var/lib/clickhouse/store/"}} - i := 0 - for { - values, ok, err := frame.Next() - - if i < 4 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - require.Equal(t, expectedRows[i][2], values[2]) - require.NotNil(t, values[3]) - require.NotNil(t, values[4]) - require.Equal(t, "", values[5]) - } else { - require.False(t, ok) - break - } - i += 1 - } - }) - - t.Run("client can get all rows for system.databases table with except", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - frame, err := clickhouseClient.ReadTable("system", "databases", []string{"data_path", "comment"}, data.OrderBy{}, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [4]string{"name", "engine", "metadata_path", "uuid"}) - }) - - t.Run("client can limit rows for system.databases", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - frame, err := clickhouseClient.ReadTable("system", "databases", []string{}, data.OrderBy{}, 1) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [6]string{"name", "engine", "data_path", "metadata_path", "uuid", "comment"}) - expectedRows := [1][3]string{{"INFORMATION_SCHEMA", "Memory", "/var/lib/clickhouse/"}} - i := 0 - for { - values, ok, err := frame.Next() - if i == 0 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - require.Equal(t, expectedRows[i][2], values[2]) - require.NotNil(t, values[3]) - require.NotNil(t, values[4]) - require.Equal(t, "", values[5]) - } else { - require.False(t, ok) - break - } - i += 1 - } - }) - - t.Run("client can order rows for system.databases", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - frame, err := clickhouseClient.ReadTable("system", "databases", []string{}, data.OrderBy{ - Column: "engine", - Order: data.Asc, - }, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [6]string{"name", "engine", "data_path", "metadata_path", "uuid", "comment"}) - expectedRows := [4][3]string{ - {"default", "Atomic", "/var/lib/clickhouse/store/"}, - {"system", "Atomic", "/var/lib/clickhouse/store/"}, - {"INFORMATION_SCHEMA", "Memory", "/var/lib/clickhouse/"}, - {"information_schema", "Memory", "/var/lib/clickhouse/"}, - } - i := 0 - for { - values, ok, err := frame.Next() - - if i < 4 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - require.Equal(t, expectedRows[i][2], values[2]) - require.NotNil(t, values[3]) - require.NotNil(t, values[4]) - require.Equal(t, "", values[5]) - } else { - require.False(t, ok) - break - } - i += 1 - } - }) -} - -func TestExecuteStatement(t *testing.T) { - t.Run("client can execute any statement", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - statement := "SELECT path, count(*) as count FROM system.disks GROUP BY path;" - frame, err := clickhouseClient.ExecuteStatement("engines", statement) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [2]string{"path", "count"}) - expectedRows := [1][2]interface{}{ - {"/var/lib/clickhouse/", uint64(1)}, - } - i := 0 - for { - values, ok, err := frame.Next() - if !ok { - require.Nil(t, err) - break - } - require.Nil(t, err) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - i++ - } - fmt.Println(i) - }) -} - -func TestVersion(t *testing.T) { - t.Run("client can read version", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - version, err := clickhouseClient.Version() - require.Nil(t, err) - require.NotEmpty(t, version) - }) -} diff --git a/programs/diagnostics/internal/platform/manager.go b/programs/diagnostics/internal/platform/manager.go deleted file mode 100644 index b4435b62ea2..00000000000 --- a/programs/diagnostics/internal/platform/manager.go +++ /dev/null @@ -1,49 +0,0 @@ -package platform - -import ( - "errors" - "sync" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" -) - -var once sync.Once -var dbInit sync.Once - -// manages all resources that collectors and outputs may wish to ensure inc. db connections - -type DBClient interface { - ReadTableNamesForDatabase(databaseName string) ([]string, error) - ReadTable(databaseName string, tableName string, excludeColumns []string, orderBy data.OrderBy, limit int64) (data.Frame, error) - ExecuteStatement(id string, statement string) (data.Frame, error) - Version() (string, error) -} - -var manager *ResourceManager - -type ResourceManager struct { - DbClient DBClient -} - -func GetResourceManager() *ResourceManager { - once.Do(func() { - manager = &ResourceManager{} - }) - return manager -} - -func (m *ResourceManager) Connect(host string, port uint16, username string, password string) error { - var err error - var clientInstance DBClient - init := false - dbInit.Do(func() { - clientInstance, err = database.NewNativeClient(host, port, username, password) - manager.DbClient = clientInstance - init = true - }) - if !init { - return errors.New("connect can only be called once") - } - return err -} diff --git a/programs/diagnostics/internal/platform/manager_test.go b/programs/diagnostics/internal/platform/manager_test.go deleted file mode 100644 index e6c50c6e505..00000000000 --- a/programs/diagnostics/internal/platform/manager_test.go +++ /dev/null @@ -1,100 +0,0 @@ -//go:build !no_docker - -package platform_test - -import ( - "context" - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/docker/go-connections/nat" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -// create a ClickHouse container -func createClickHouseContainer(t *testing.T, ctx context.Context) (testcontainers.Container, nat.Port) { - cwd, err := os.Getwd() - if err != nil { - fmt.Println("unable to read current directory", err) - os.Exit(1) - } - // for now, we test against a hardcoded database-server version but we should make this a property - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../testdata/docker/admin.xml"), - }, - Target: "/etc/clickhouse-server/users.d/admin.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, err := clickhouseContainer.MappedPort(ctx, "9000") - if err != nil { - // can't test without a port - panic(err) - } - - return clickhouseContainer, p -} - -func TestConnect(t *testing.T) { - t.Run("can only connect once", func(t *testing.T) { - ctx := context.Background() - - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - t.Setenv("CLICKHOUSE_DB_PORT", mappedPort.Port()) - - port := mappedPort.Int() - - // get before connection - manager := platform.GetResourceManager() - require.Nil(t, manager.DbClient) - // init connection - err := manager.Connect("localhost", uint16(port), "", "") - require.Nil(t, err) - require.NotNil(t, manager.DbClient) - // try and re-fetch connection - err = manager.Connect("localhost", uint16(port), "", "") - require.NotNil(t, err) - require.Equal(t, "connect can only be called once", err.Error()) - }) - -} - -func TestGetResourceManager(t *testing.T) { - t.Run("get resource manager", func(t *testing.T) { - manager := platform.GetResourceManager() - require.NotNil(t, manager) - manager2 := platform.GetResourceManager() - require.NotNil(t, manager2) - require.Equal(t, &manager, &manager2) - }) - -} diff --git a/programs/diagnostics/internal/platform/test/data.go b/programs/diagnostics/internal/platform/test/data.go deleted file mode 100644 index 7710e9a69a1..00000000000 --- a/programs/diagnostics/internal/platform/test/data.go +++ /dev/null @@ -1,166 +0,0 @@ -package test - -import ( - "fmt" - "sort" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" -) - -type fakeClickhouseClient struct { - tables map[string][]string - QueryResponses map[string]*FakeDataFrame -} - -func NewFakeClickhouseClient(tables map[string][]string) fakeClickhouseClient { - queryResponses := make(map[string]*FakeDataFrame) - return fakeClickhouseClient{ - tables: tables, - QueryResponses: queryResponses, - } -} - -func (f fakeClickhouseClient) ReadTableNamesForDatabase(databaseName string) ([]string, error) { - if _, ok := f.tables[databaseName]; ok { - return f.tables[databaseName], nil - } - return nil, fmt.Errorf("database %s does not exist", databaseName) -} - -func (f fakeClickhouseClient) ReadTable(databaseName string, tableName string, excludeColumns []string, orderBy data.OrderBy, limit int64) (data.Frame, error) { - - exceptClause := "" - if len(excludeColumns) > 0 { - exceptClause = fmt.Sprintf("EXCEPT(%s) ", strings.Join(excludeColumns, ",")) - } - limitClause := "" - if limit >= 0 { - limitClause = fmt.Sprintf(" LIMIT %d", limit) - } - query := fmt.Sprintf("SELECT * %sFROM %s.%s%s%s", exceptClause, databaseName, tableName, orderBy.String(), limitClause) - frame, error := f.ExecuteStatement(fmt.Sprintf("read_table_%s.%s", databaseName, tableName), query) - if error != nil { - return frame, error - } - fFrame := *(frame.(*FakeDataFrame)) - fFrame = fFrame.FilterColumns(excludeColumns) - fFrame = fFrame.Order(orderBy) - fFrame = fFrame.Limit(limit) - return fFrame, nil -} - -func (f fakeClickhouseClient) ExecuteStatement(id string, statement string) (data.Frame, error) { - if frame, ok := f.QueryResponses[statement]; ok { - return frame, nil - } - return FakeDataFrame{}, errors.New(fmt.Sprintf("No recorded response for %s", statement)) -} - -func (f fakeClickhouseClient) Version() (string, error) { - return "21.12.3", nil -} - -func (f fakeClickhouseClient) Reset() { - for key, frame := range f.QueryResponses { - frame.Reset() - f.QueryResponses[key] = frame - } -} - -type FakeDataFrame struct { - i *int - Rows [][]interface{} - ColumnNames []string - name string -} - -func NewFakeDataFrame(name string, columns []string, rows [][]interface{}) FakeDataFrame { - i := 0 - return FakeDataFrame{ - i: &i, - Rows: rows, - ColumnNames: columns, - name: name, - } -} - -func (f FakeDataFrame) Next() ([]interface{}, bool, error) { - if len(f.Rows) == *(f.i) { - return nil, false, nil - } - value := f.Rows[*f.i] - *f.i++ - return value, true, nil -} - -func (f FakeDataFrame) Columns() []string { - return f.ColumnNames -} - -func (f FakeDataFrame) Name() string { - return f.name -} - -func (f *FakeDataFrame) Reset() { - i := 0 - f.i = &i -} - -func (f FakeDataFrame) FilterColumns(excludeColumns []string) FakeDataFrame { - // get columns we can remove - rColumns := utils.Intersection(f.ColumnNames, excludeColumns) - rIndexes := make([]int, len(rColumns)) - // find the indexes of the columns to remove - for i, column := range rColumns { - rIndexes[i] = utils.IndexOf(f.ColumnNames, column) - } - newRows := make([][]interface{}, len(f.Rows)) - for r, row := range f.Rows { - newRow := row - for i, index := range rIndexes { - newRow = utils.Remove(newRow, index-i) - } - newRows[r] = newRow - } - f.Rows = newRows - f.ColumnNames = utils.Distinct(f.ColumnNames, excludeColumns) - return f -} - -func (f FakeDataFrame) Limit(rowLimit int64) FakeDataFrame { - if rowLimit >= 0 { - if int64(len(f.Rows)) > rowLimit { - f.Rows = f.Rows[:rowLimit] - } - } - return f -} - -func (f FakeDataFrame) Order(orderBy data.OrderBy) FakeDataFrame { - if orderBy.Column == "" { - return f - } - cIndex := utils.IndexOf(f.ColumnNames, orderBy.Column) - sort.Slice(f.Rows, func(i, j int) bool { - left := f.Rows[i][cIndex] - right := f.Rows[j][cIndex] - if iLeft, ok := left.(int); ok { - if orderBy.Order == data.Asc { - return iLeft < right.(int) - } - return iLeft > right.(int) - } else { - // we aren't a full db - revert to string order - sLeft := left.(string) - sRight := right.(string) - if orderBy.Order == data.Asc { - return sLeft < sRight - } - return sLeft > sRight - } - }) - return f -} diff --git a/programs/diagnostics/internal/platform/test/env.go b/programs/diagnostics/internal/platform/test/env.go deleted file mode 100644 index 36b03772ab0..00000000000 --- a/programs/diagnostics/internal/platform/test/env.go +++ /dev/null @@ -1,16 +0,0 @@ -package test - -import "os" - -const defaultClickHouseVersion = "latest" - -func GetClickHouseTestVersion() string { - return GetEnv("CLICKHOUSE_VERSION", defaultClickHouseVersion) -} - -func GetEnv(key, fallback string) string { - if value, ok := os.LookupEnv(key); ok { - return value - } - return fallback -} diff --git a/programs/diagnostics/internal/platform/utils/file.go b/programs/diagnostics/internal/platform/utils/file.go deleted file mode 100644 index 71af4b32658..00000000000 --- a/programs/diagnostics/internal/platform/utils/file.go +++ /dev/null @@ -1,95 +0,0 @@ -package utils - -import ( - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -func FileExists(name string) (bool, error) { - f, err := os.Stat(name) - if err == nil { - if !f.IsDir() { - return true, nil - } - return false, fmt.Errorf("%s is a directory", name) - } - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - return false, err -} - -func DirExists(name string) (bool, error) { - f, err := os.Stat(name) - if err == nil { - if f.IsDir() { - return true, nil - } - return false, fmt.Errorf("%s is a file", name) - } - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - return false, err -} - -func CopyFile(sourceFilename string, destFilename string) error { - exists, err := FileExists(sourceFilename) - if err != nil { - return err - } - if !exists { - return fmt.Errorf("%s does not exist", sourceFilename) - } - source, err := os.Open(sourceFilename) - if err != nil { - return err - } - defer source.Close() - destDir := filepath.Dir(destFilename) - if err := os.MkdirAll(destDir, os.ModePerm); err != nil { - return errors.Wrapf(err, "unable to create directory %s", destDir) - } - - destination, err := os.Create(destFilename) - if err != nil { - return err - } - defer destination.Close() - _, err = io.Copy(destination, source) - return err -} - -// patterns passed are an OR - any can be satisfied and the file will be listed - -func ListFilesInDirectory(directory string, patterns []string) ([]string, []error) { - var files []string - exists, err := DirExists(directory) - if err != nil { - return files, []error{err} - } - if !exists { - return files, []error{fmt.Errorf("directory %s does not exist", directory)} - } - var pathErrors []error - _ = filepath.Walk(directory, func(path string, info fs.FileInfo, err error) error { - if err != nil { - pathErrors = append(pathErrors, err) - } else if !info.IsDir() { - for _, pattern := range patterns { - if matched, err := filepath.Match(pattern, filepath.Base(path)); err != nil { - pathErrors = append(pathErrors, err) - } else if matched { - files = append(files, path) - } - } - } - return nil - }) - return files, pathErrors -} diff --git a/programs/diagnostics/internal/platform/utils/file_test.go b/programs/diagnostics/internal/platform/utils/file_test.go deleted file mode 100644 index 8d0430090c9..00000000000 --- a/programs/diagnostics/internal/platform/utils/file_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package utils_test - -import ( - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" -) - -func TestFileExists(t *testing.T) { - t.Run("returns true for file", func(t *testing.T) { - tempDir := t.TempDir() - filepath := path.Join(tempDir, "random.txt") - _, err := os.Create(filepath) - require.Nil(t, err) - exists, err := utils.FileExists(filepath) - require.True(t, exists) - require.Nil(t, err) - }) - - t.Run("doesn't return true for not existence file", func(t *testing.T) { - tempDir := t.TempDir() - file := path.Join(tempDir, "random.txt") - exists, err := utils.FileExists(file) - require.False(t, exists) - require.Nil(t, err) - }) - - t.Run("doesn't return true for directory", func(t *testing.T) { - tempDir := t.TempDir() - exists, err := utils.FileExists(tempDir) - require.False(t, exists) - require.NotNil(t, err) - require.Equal(t, fmt.Sprintf("%s is a directory", tempDir), err.Error()) - }) -} - -func TestDirExists(t *testing.T) { - t.Run("doesn't return true for file", func(t *testing.T) { - tempDir := t.TempDir() - filepath := path.Join(tempDir, "random.txt") - _, err := os.Create(filepath) - require.Nil(t, err) - exists, err := utils.DirExists(filepath) - require.False(t, exists) - require.NotNil(t, err) - require.Equal(t, fmt.Sprintf("%s is a file", filepath), err.Error()) - }) - - t.Run("returns true for directory", func(t *testing.T) { - tempDir := t.TempDir() - exists, err := utils.DirExists(tempDir) - require.True(t, exists) - require.Nil(t, err) - }) - - t.Run("doesn't return true random directory", func(t *testing.T) { - exists, err := utils.FileExists(fmt.Sprintf("%d", utils.MakeTimestamp())) - require.False(t, exists) - require.Nil(t, err) - }) -} - -func TestCopyFile(t *testing.T) { - t.Run("can copy file", func(t *testing.T) { - tempDir := t.TempDir() - sourcePath := path.Join(tempDir, "random.txt") - _, err := os.Create(sourcePath) - require.Nil(t, err) - destPath := path.Join(tempDir, "random-2.txt") - err = utils.CopyFile(sourcePath, destPath) - require.Nil(t, err) - }) - - t.Run("can copy nested file", func(t *testing.T) { - tempDir := t.TempDir() - sourcePath := path.Join(tempDir, "random.txt") - _, err := os.Create(sourcePath) - require.Nil(t, err) - destPath := path.Join(tempDir, "sub_dir", "random-2.txt") - err = utils.CopyFile(sourcePath, destPath) - require.Nil(t, err) - }) - - t.Run("fails when file does not exist", func(t *testing.T) { - tempDir := t.TempDir() - sourcePath := path.Join(tempDir, "random.txt") - destPath := path.Join(tempDir, "random-2.txt") - err := utils.CopyFile(sourcePath, destPath) - require.NotNil(t, err) - require.Equal(t, fmt.Sprintf("%s does not exist", sourcePath), err.Error()) - }) -} - -func TestListFilesInDirectory(t *testing.T) { - tempDir := t.TempDir() - files := make([]string, 5) - for i := 0; i < 5; i++ { - fileDir := path.Join(tempDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - ext := ".txt" - if i%2 == 0 { - ext = ".csv" - } - filepath := path.Join(fileDir, fmt.Sprintf("random-%d%s", i, ext)) - files[i] = filepath - _, err = os.Create(filepath) - require.Nil(t, err) - } - - t.Run("can list all files", func(t *testing.T) { - mFiles, errs := utils.ListFilesInDirectory(tempDir, []string{"*"}) - require.Len(t, mFiles, 5) - require.Empty(t, errs) - }) - - t.Run("can list by extension", func(t *testing.T) { - mFiles, errs := utils.ListFilesInDirectory(tempDir, []string{"*.csv"}) - require.Len(t, mFiles, 3) - require.Empty(t, errs) - require.ElementsMatch(t, []string{files[0], files[2], files[4]}, mFiles) - }) - - t.Run("can list on multiple extensions files", func(t *testing.T) { - mFiles, errs := utils.ListFilesInDirectory(tempDir, []string{"*.csv", "*.txt"}) - require.Len(t, mFiles, 5) - require.Empty(t, errs) - }) - -} diff --git a/programs/diagnostics/internal/platform/utils/process.go b/programs/diagnostics/internal/platform/utils/process.go deleted file mode 100644 index 7b27c215eea..00000000000 --- a/programs/diagnostics/internal/platform/utils/process.go +++ /dev/null @@ -1,49 +0,0 @@ -package utils - -import ( - "github.com/elastic/gosigar" - "strings" -) - -func FindClickHouseProcesses() ([]gosigar.ProcArgs, error) { - pids := gosigar.ProcList{} - err := pids.Get() - if err != nil { - return nil, err - } - var clickhousePs []gosigar.ProcArgs - for _, pid := range pids.List { - args := gosigar.ProcArgs{} - if err := args.Get(pid); err != nil { - continue - } - if len(args.List) > 0 { - if strings.Contains(args.List[0], "clickhouse-server") { - clickhousePs = append(clickhousePs, args) - } - } - } - return clickhousePs, nil -} - -func FindConfigsFromClickHouseProcesses() ([]string, error) { - clickhouseProcesses, err := FindClickHouseProcesses() - if err != nil { - return nil, err - } - var configs []string - if len(clickhouseProcesses) > 0 { - // we have candidate matches - for _, ps := range clickhouseProcesses { - for _, arg := range ps.List { - if strings.Contains(arg, "--config") { - configFile := strings.ReplaceAll(arg, "--config-file=", "") - // containers receive config with --config - configFile = strings.ReplaceAll(configFile, "--config=", "") - configs = append(configs, configFile) - } - } - } - } - return configs, err -} diff --git a/programs/diagnostics/internal/platform/utils/process_test.go b/programs/diagnostics/internal/platform/utils/process_test.go deleted file mode 100644 index 9baaa559752..00000000000 --- a/programs/diagnostics/internal/platform/utils/process_test.go +++ /dev/null @@ -1,97 +0,0 @@ -//go:build !no_docker - -package utils_test - -import ( - "context" - "fmt" - "io" - "os" - "path" - "strings" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -func getProcessesInContainer(t *testing.T, container testcontainers.Container) ([]string, error) { - result, reader, err := container.Exec(context.Background(), []string{"ps", "-aux"}) - if err != nil { - return nil, err - } - require.Zero(t, result) - require.NotNil(t, reader) - - b, err := io.ReadAll(reader) - if err != nil { - return nil, err - } - require.NotNil(t, b) - - lines := strings.Split(string(b), "\n") - - // discard PS header - return lines[1:], nil -} - -func TestFindClickHouseProcessesAndConfigs(t *testing.T) { - - t.Run("can find ClickHouse processes and configs", func(t *testing.T) { - // create a ClickHouse container - ctx := context.Background() - cwd, err := os.Getwd() - if err != nil { - fmt.Println("unable to read current directory", err) - os.Exit(1) - } - - // run a ClickHouse container that guarantees that it runs only for the duration of the test - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, _ := clickhouseContainer.MappedPort(ctx, "9000") - - t.Setenv("CLICKHOUSE_DB_PORT", p.Port()) - - defer clickhouseContainer.Terminate(ctx) //nolint - - lines, err := getProcessesInContainer(t, clickhouseContainer) - require.Nil(t, err) - require.NotEmpty(t, lines) - - for _, line := range lines { - parts := strings.Fields(line) - if len(parts) < 11 { - continue - } - if !strings.Contains(parts[10], "clickhouse-server") { - continue - } - - require.Equal(t, "/usr/bin/clickhouse-server", parts[10]) - require.Equal(t, "--config-file=/etc/clickhouse-server/config.xml", parts[11]) - } - }) -} diff --git a/programs/diagnostics/internal/platform/utils/slices.go b/programs/diagnostics/internal/platform/utils/slices.go deleted file mode 100644 index cf5a5f97ce8..00000000000 --- a/programs/diagnostics/internal/platform/utils/slices.go +++ /dev/null @@ -1,68 +0,0 @@ -package utils - -// Intersection of elements in s1 and s2 -func Intersection(s1, s2 []string) (inter []string) { - hash := make(map[string]bool) - for _, e := range s1 { - hash[e] = false - } - for _, e := range s2 { - // If elements present in the hashmap then append intersection list. - if val, ok := hash[e]; ok { - if !val { - // only add once - inter = append(inter, e) - hash[e] = true - } - } - } - return inter -} - -// Distinct returns elements in s1, not in s2 -func Distinct(s1, s2 []string) (distinct []string) { - hash := make(map[string]bool) - for _, e := range s2 { - hash[e] = true - } - for _, e := range s1 { - if _, ok := hash[e]; !ok { - distinct = append(distinct, e) - } - } - return distinct -} - -// Unique func Unique(s1 []string) (unique []string) returns unique elements in s1 -func Unique(s1 []string) (unique []string) { - hash := make(map[string]bool) - for _, e := range s1 { - if _, ok := hash[e]; !ok { - unique = append(unique, e) - } - hash[e] = true - } - return unique -} - -func Contains(s []string, e string) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} - -func IndexOf(s []string, e string) int { - for i, a := range s { - if a == e { - return i - } - } - return -1 -} - -func Remove(slice []interface{}, s int) []interface{} { - return append(slice[:s], slice[s+1:]...) -} diff --git a/programs/diagnostics/internal/platform/utils/slices_test.go b/programs/diagnostics/internal/platform/utils/slices_test.go deleted file mode 100644 index ea5c1c81dcc..00000000000 --- a/programs/diagnostics/internal/platform/utils/slices_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package utils_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" -) - -func TestIntersection(t *testing.T) { - t.Run("can perform intersection", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - setB := []string{"A", "B", "F", "C", "G"} - setC := utils.Intersection(setA, setB) - require.Len(t, setC, 2) - require.ElementsMatch(t, []string{"A", "C"}, setC) - }) -} - -func TestDistinct(t *testing.T) { - t.Run("can perform distinct", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - setB := []string{"A", "B", "F", "C", "G"} - setC := utils.Distinct(setA, setB) - require.Len(t, setC, 3) - require.ElementsMatch(t, []string{"b", "D", "E"}, setC) - }) - - t.Run("can perform distinct on empty", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - var setB []string - setC := utils.Distinct(setA, setB) - require.Len(t, setC, 5) - require.ElementsMatch(t, []string{"A", "b", "C", "D", "E"}, setC) - }) -} - -func TestContains(t *testing.T) { - t.Run("can perform contains", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - require.True(t, utils.Contains(setA, "A")) - require.True(t, utils.Contains(setA, "b")) - require.True(t, utils.Contains(setA, "C")) - require.True(t, utils.Contains(setA, "D")) - require.True(t, utils.Contains(setA, "E")) - require.False(t, utils.Contains(setA, "B")) - }) -} - -func TestUnique(t *testing.T) { - - t.Run("can perform unique", func(t *testing.T) { - setA := []string{"A", "b", "D", "D", "E", "E", "A"} - setC := utils.Unique(setA) - require.Len(t, setC, 4) - require.ElementsMatch(t, []string{"A", "b", "D", "E"}, setC) - }) - - t.Run("can perform unique on empty", func(t *testing.T) { - var setA []string - setC := utils.Unique(setA) - require.Len(t, setC, 0) - }) -} diff --git a/programs/diagnostics/internal/platform/utils/time.go b/programs/diagnostics/internal/platform/utils/time.go deleted file mode 100644 index 622e92b873a..00000000000 --- a/programs/diagnostics/internal/platform/utils/time.go +++ /dev/null @@ -1,7 +0,0 @@ -package utils - -import "time" - -func MakeTimestamp() int64 { - return time.Now().UnixNano() / int64(time.Millisecond) -} diff --git a/programs/diagnostics/internal/runner.go b/programs/diagnostics/internal/runner.go deleted file mode 100644 index 9386a1d178b..00000000000 --- a/programs/diagnostics/internal/runner.go +++ /dev/null @@ -1,115 +0,0 @@ -package internal - -import ( - c "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - o "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -type runConfiguration struct { - id string - host string - port uint16 - username string - password string - output string - collectors []string - collectorConfigs map[string]config.Configuration - outputConfig config.Configuration -} - -func NewRunConfiguration(id string, host string, port uint16, username string, password string, output string, outputConfig config.Configuration, - collectors []string, collectorConfigs map[string]config.Configuration) *runConfiguration { - config := runConfiguration{ - id: id, - host: host, - port: port, - username: username, - password: password, - collectors: collectors, - output: output, - collectorConfigs: collectorConfigs, - outputConfig: outputConfig, - } - return &config -} - -func Capture(config *runConfiguration) { - bundles, err := collect(config) - if err != nil { - log.Fatal().Err(err).Msg("unable to perform collection") - } - log.Info().Msgf("collectors initialized") - if err = output(config, bundles); err != nil { - log.Fatal().Err(err).Msg("unable to create output") - } - log.Info().Msgf("bundle export complete") -} - -func collect(config *runConfiguration) (map[string]*data.DiagnosticBundle, error) { - resourceManager := platform.GetResourceManager() - err := resourceManager.Connect(config.host, config.port, config.username, config.password) - if err != nil { - // if we can't connect this is fatal - log.Fatal().Err(err).Msg("Unable to connect to database") - } - //grab the required connectors - we pass what we can - bundles := make(map[string]*data.DiagnosticBundle) - log.Info().Msgf("connection established") - //these store our collection errors and will be output in the bundle - var collectorErrors [][]interface{} - for _, collectorName := range config.collectors { - collectorConfig := config.collectorConfigs[collectorName] - log.Info().Msgf("initializing %s collector", collectorName) - collector, err := c.GetCollectorByName(collectorName) - if err != nil { - log.Error().Err(err).Msgf("Unable to fetch collector %s", collectorName) - collectorErrors = append(collectorErrors, []interface{}{err.Error()}) - continue - } - bundle, err := collector.Collect(collectorConfig) - if err != nil { - log.Error().Err(err).Msgf("Error in collector %s", collectorName) - collectorErrors = append(collectorErrors, []interface{}{err.Error()}) - // this indicates a fatal error in the collector - continue - } - for _, fError := range bundle.Errors.Errors { - err = errors.Wrapf(fError, "Failure to collect frame in collector %s", collectorName) - collectorErrors = append(collectorErrors, []interface{}{err.Error()}) - log.Warn().Msg(err.Error()) - } - bundles[collectorName] = bundle - } - bundles["diag_trace"] = buildTraceBundle(collectorErrors) - return bundles, nil -} - -func output(config *runConfiguration, bundles map[string]*data.DiagnosticBundle) error { - log.Info().Msgf("attempting to export bundle using %s output...", config.output) - output, err := o.GetOutputByName(config.output) - if err != nil { - return err - } - frameErrors, err := output.Write(config.id, bundles, config.outputConfig) - // we report over failing hard on frame errors - up to the output to determine what is fatal via error - for _, fError := range frameErrors.Errors { - log.Warn().Msgf("failure to write frame in output %s - %s", config.output, fError) - } - return err -} - -func buildTraceBundle(collectorErrors [][]interface{}) *data.DiagnosticBundle { - errorBundle := data.DiagnosticBundle{ - Frames: map[string]data.Frame{ - "errors": data.NewMemoryFrame("errors", []string{"errors"}, collectorErrors), - }, - Errors: data.FrameErrors{}, - } - // add any other metrics from collection - return &errorBundle -} diff --git a/programs/diagnostics/internal/runner_test.go b/programs/diagnostics/internal/runner_test.go deleted file mode 100644 index 2369f8b3007..00000000000 --- a/programs/diagnostics/internal/runner_test.go +++ /dev/null @@ -1,130 +0,0 @@ -//go:build !no_docker - -package internal_test - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -// Execute a full default capture, with simple output, and check if a bundle is produced and it's not empty -func TestCapture(t *testing.T) { - // create a ClickHouse container - ctx := context.Background() - cwd, err := os.Getwd() - - if err != nil { - // can't test without container - panic(err) - } - // for now, we test against a hardcoded database-server version but we should make this a property - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../testdata/docker/admin.xml"), - }, - Target: "/etc/clickhouse-server/users.d/admin.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, _ := clickhouseContainer.MappedPort(ctx, "9000") - - t.Setenv("CLICKHOUSE_DB_PORT", p.Port()) - defer clickhouseContainer.Terminate(ctx) //nolint - - tmrDir := t.TempDir() - port := p.Int() - - // test a simple output exists - _, err = outputs.GetOutputByName("simple") - require.Nil(t, err) - // this relies on the simple out not changing its params - test will likely fail if so - outputConfig := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: tmrDir, - Param: config.NewParam("directory", "Directory in which to create dump. Defaults to the current directory.", false), - }, - config.StringOptions{ - Value: "csv", - Options: []string{"csv"}, - Param: config.NewParam("format", "Format of exported files", false), - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Don't compress output to an archive", false), - }, - }, - } - // test default collectors - collectorNames := collectors.GetCollectorNames(true) - // grab all configs - only default will be used because of collectorNames - collectorConfigs, err := collectors.BuildConfigurationOptions() - require.Nil(t, err) - conf := internal.NewRunConfiguration("random", "localhost", uint16(port), "", "", "simple", outputConfig, collectorNames, collectorConfigs) - internal.Capture(conf) - outputDir := path.Join(tmrDir, "random") - _, err = os.Stat(outputDir) - require.Nil(t, err) - require.True(t, !os.IsNotExist(err)) - files, err := ioutil.ReadDir(outputDir) - require.Nil(t, err) - require.Len(t, files, 1) - outputDir = path.Join(outputDir, files[0].Name()) - // check we have a folder per collector i.e. collectorNames + diag_trace - files, err = ioutil.ReadDir(outputDir) - require.Nil(t, err) - require.Len(t, files, len(collectorNames)+1) - expectedFolders := append(collectorNames, "diag_trace") - for _, file := range files { - require.True(t, file.IsDir()) - utils.Contains(expectedFolders, file.Name()) - } - // we don't test the specific collector outputs but make sure something was written to system - systemFolder := path.Join(outputDir, "system") - files, err = ioutil.ReadDir(systemFolder) - require.Nil(t, err) - require.Greater(t, len(files), 0) - // test diag_trace - diagFolder := path.Join(outputDir, "diag_trace") - files, err = ioutil.ReadDir(diagFolder) - require.Nil(t, err) - require.Equal(t, 1, len(files)) - require.FileExists(t, path.Join(diagFolder, "errors.csv")) -} diff --git a/programs/diagnostics/testdata/configs/include/xml/server-include.xml b/programs/diagnostics/testdata/configs/include/xml/server-include.xml deleted file mode 100644 index 30e6587c935..00000000000 --- a/programs/diagnostics/testdata/configs/include/xml/server-include.xml +++ /dev/null @@ -1,8 +0,0 @@ - - 5000000 - - - - - 9008 - \ No newline at end of file diff --git a/programs/diagnostics/testdata/configs/include/xml/user-include.xml b/programs/diagnostics/testdata/configs/include/xml/user-include.xml deleted file mode 100644 index b12b34a56bb..00000000000 --- a/programs/diagnostics/testdata/configs/include/xml/user-include.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - ::/0 - - default - default - REPLACE_ME - 1 - - - - ::/0 - - default - default - REPLACE_ME - 1 - - diff --git a/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml b/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml deleted file mode 100644 index 903d7b6f733..00000000000 --- a/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml +++ /dev/null @@ -1 +0,0 @@ -network_max: 5000000 diff --git a/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml b/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml deleted file mode 100644 index 23b592507fa..00000000000 --- a/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml +++ /dev/null @@ -1,7 +0,0 @@ -test_user: - password: 'REPLACE_ME' - networks: - ip: '::/0' - profile: default - quota: default - access_management: 1 diff --git a/programs/diagnostics/testdata/configs/xml/config.xml b/programs/diagnostics/testdata/configs/xml/config.xml deleted file mode 100644 index eb7c70cf498..00000000000 --- a/programs/diagnostics/testdata/configs/xml/config.xml +++ /dev/null @@ -1,1195 +0,0 @@ - - - ../include/xml/server-include.xml - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - - 1000M - 10 - - - - - - - - - - - - - - - - - 8123 - - - 9000 - - - 9004 - - - 9005 - - - - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4096 - - - 3 - - - - - false - - - /path/to/ssl_cert_file - /path/to/ssl_key_file - - - false - - - /path/to/ssl_ca_cert_file - - - deflate - - - medium - - - -1 - -1 - - - false - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 100 - - - 0 - - - - 10000 - - - 0.9 - - - 4194304 - - - 0 - - - - - - 8589934592 - - - 5368709120 - - - - 1000 - - - 134217728 - - - 10000 - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - - - - /var/lib/clickhouse/user_files/ - - - - - - - - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - - - default - - - - - - - - - - - - default - - - - - - - - - true - - - false - - ' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - clickhouse-jdbc-bridge & - - * [CentOS/RHEL] - export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge - export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - clickhouse-jdbc-bridge & - - Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - ]]> - - - - - - - - REPLACE_ME - - - - - - - - localhost - 9000 - - - - - - - - - - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - - - - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - query_views_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - system - metric_log
- 7500 - 1000 - - - - - system - asynchronous_metric_log
- - 7000 - - - - - - - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - - system - opentelemetry_span_log
- 7500 - - - - - - system - crash_log
- - - 1000 - - - - - system - session_log
- - toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - - - - - *_dictionary.xml - - - *_function.xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - - hide encrypt/decrypt arguments - ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) - - \1(???) - - - - - - - - - - false - - false - - - https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 - - - - - - - - - - - s3 - https://storage.yandexcloud.net/my-bucket/root-path/ - REPLACE_ME - REPLACE_ME - -
Authorization: Bearer SOME-TOKEN
- your_base64_encoded_customer_key - - REPLACE_ME - REPLACE_ME - true - - http://proxy1 - http://proxy2 - - 10000 - 5000 - 10 - 4 - 1000 - /var/lib/clickhouse/disks/s3/ - false - - - - diff --git a/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml b/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml deleted file mode 100644 index 242a6a4b02e..00000000000 --- a/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - REPLACE_ME - - - \ No newline at end of file diff --git a/programs/diagnostics/testdata/configs/xml/users.xml b/programs/diagnostics/testdata/configs/xml/users.xml deleted file mode 100644 index cd5f17e922e..00000000000 --- a/programs/diagnostics/testdata/configs/xml/users.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - ../include/xml/user-include.xml - - - - - - 10000000000 - - random - 1 - - - - 1 - - - - - - - - - - REPLACE_ME - - ::/0 - - - default - - default - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/programs/diagnostics/testdata/configs/yaml/config.yaml b/programs/diagnostics/testdata/configs/yaml/config.yaml deleted file mode 100644 index e577a99e675..00000000000 --- a/programs/diagnostics/testdata/configs/yaml/config.yaml +++ /dev/null @@ -1,927 +0,0 @@ -# This is an example of a configuration file "config.xml" rewritten in YAML -# You can read this documentation for detailed information about YAML configuration: -# https://clickhouse.com/docs/en/operations/configuration-files/ - -# NOTE: User and query level settings are set up in "users.yaml" file. -# If you have accidentally specified user-level settings here, server won't start. -# You can either move the settings to the right place inside "users.xml" file -# or add skip_check_for_incorrect_settings: 1 here. -include_from: "../include/yaml/server-include.yaml" -logger: - # Possible levels [1]: - # - none (turns off logging) - # - fatal - # - critical - # - error - # - warning - # - notice - # - information - # - debug - # - trace - # [1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114 - level: trace - log: /var/log/clickhouse-server/clickhouse-server.log - errorlog: /var/log/clickhouse-server/clickhouse-server.err.log - # Rotation policy - # See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85 - size: 1000M - count: 10 - # console: 1 - # Default behavior is autodetection (log to console if not daemon mode and is tty) - - # Per level overrides (legacy): - # For example to suppress logging of the ConfigReloader you can use: - # NOTE: levels.logger is reserved, see below. - # levels: - # ConfigReloader: none - - # Per level overrides: - # For example to suppress logging of the RBAC for default user you can use: - # (But please note that the logger name maybe changed from version to version, even after minor upgrade) - # levels: - # - logger: - # name: 'ContextAccess (default)' - # level: none - # - logger: - # name: 'DatabaseOrdinary (test)' - # level: none - -# It is the name that will be shown in the clickhouse-client. -# By default, anything with "production" will be highlighted in red in query prompt. -# display_name: production - -# Port for HTTP API. See also 'https_port' for secure connections. -# This interface is also used by ODBC and JDBC drivers (DataGrip, Dbeaver, ...) -# and by most of web interfaces (embedded UI, Grafana, Redash, ...). -http_port: 8123 - -# Port for interaction by native protocol with: -# - clickhouse-client and other native ClickHouse tools (clickhouse-benchmark); -# - clickhouse-server with other clickhouse-servers for distributed query processing; -# - ClickHouse drivers and applications supporting native protocol -# (this protocol is also informally called as "the TCP protocol"); -# See also 'tcp_port_secure' for secure connections. -tcp_port: 9000 - -# Compatibility with MySQL protocol. -# ClickHouse will pretend to be MySQL for applications connecting to this port. -mysql_port: 9004 - -# Compatibility with PostgreSQL protocol. -# ClickHouse will pretend to be PostgreSQL for applications connecting to this port. -postgresql_port: 9005 - -# HTTP API with TLS (HTTPS). -# You have to configure certificate to enable this interface. -# See the openSSL section below. -# https_port: 8443 - -# Native interface with TLS. -# You have to configure certificate to enable this interface. -# See the openSSL section below. -# tcp_port_secure: 9440 - -# Native interface wrapped with PROXYv1 protocol -# PROXYv1 header sent for every connection. -# ClickHouse will extract information about proxy-forwarded client address from the header. -# tcp_with_proxy_port: 9011 - -# Port for communication between replicas. Used for data exchange. -# It provides low-level data access between servers. -# This port should not be accessible from untrusted networks. -# See also 'interserver_http_credentials'. -# Data transferred over connections to this port should not go through untrusted networks. -# See also 'interserver_https_port'. -interserver_http_port: 9009 - -# Port for communication between replicas with TLS. -# You have to configure certificate to enable this interface. -# See the openSSL section below. -# See also 'interserver_http_credentials'. -# interserver_https_port: 9010 - -# Hostname that is used by other replicas to request this server. -# If not specified, than it is determined analogous to 'hostname -f' command. -# This setting could be used to switch replication to another network interface -# (the server may be connected to multiple networks via multiple addresses) -# interserver_http_host: example.yandex.ru - -# You can specify credentials for authenthication between replicas. -# This is required when interserver_https_port is accessible from untrusted networks, -# and also recommended to avoid SSRF attacks from possibly compromised services in your network. -# interserver_http_credentials: -# user: interserver -# password: '' - -# Listen specified address. -# Use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere. -# Notes: -# If you open connections from wildcard address, make sure that at least one of the following measures applied: -# - server is protected by firewall and not accessible from untrusted networks; -# - all users are restricted to subset of network addresses (see users.xml); -# - all users have strong passwords, only secure (TLS) interfaces are accessible, or connections are only made via TLS interfaces. -# - users without password have readonly access. -# See also: https://www.shodan.io/search?query=clickhouse -# listen_host: '::' - -# Same for hosts without support for IPv6: -# listen_host: 0.0.0.0 - -# Default values - try listen localhost on IPv4 and IPv6. -# listen_host: '::1' -# listen_host: 127.0.0.1 - -# Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen. -# listen_try: 0 - -# Allow multiple servers to listen on the same address:port. This is not recommended. -# listen_reuse_port: 0 - -# listen_backlog: 64 -max_connections: 4096 - -# For 'Connection: keep-alive' in HTTP 1.1 -keep_alive_timeout: 3 - -# gRPC protocol (see src/Server/grpc_protos/clickhouse_grpc.proto for the API) -# grpc_port: 9100 -grpc: - enable_ssl: false - - # The following two files are used only if enable_ssl=1 - ssl_cert_file: /path/to/ssl_cert_file - ssl_key_file: /path/to/ssl_key_file - - # Whether server will request client for a certificate - ssl_require_client_auth: false - - # The following file is used only if ssl_require_client_auth=1 - ssl_ca_cert_file: /path/to/ssl_ca_cert_file - - # Default compression algorithm (applied if client doesn't specify another algorithm). - # Supported algorithms: none, deflate, gzip, stream_gzip - compression: deflate - - # Default compression level (applied if client doesn't specify another level). - # Supported levels: none, low, medium, high - compression_level: medium - - # Send/receive message size limits in bytes. -1 means unlimited - max_send_message_size: -1 - max_receive_message_size: -1 - - # Enable if you want very detailed logs - verbose_logs: false - -# Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -openSSL: - server: - # Used for https server AND secure tcp port - # openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt - certificateFile: /etc/clickhouse-server/server.crt - privateKeyFile: /etc/clickhouse-server/server.key - - # dhparams are optional. You can delete the dhParamsFile: element. - # To generate dhparams, use the following command: - # openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 - # Only file format with BEGIN DH PARAMETERS is supported. - dhParamsFile: /etc/clickhouse-server/dhparam.pem - verificationMode: none - loadDefaultCAFile: true - cacheSessions: true - disableProtocols: 'sslv2,sslv3' - preferServerCiphers: true - client: - # Used for connecting to https dictionary source and secured Zookeeper communication - loadDefaultCAFile: true - cacheSessions: true - disableProtocols: 'sslv2,sslv3' - preferServerCiphers: true - - # Use for self-signed: verificationMode: none - invalidCertificateHandler: - # Use for self-signed: name: AcceptCertificateHandler - name: RejectCertificateHandler - -# Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 -# http_server_default_response: |- -#
- -# Maximum number of concurrent queries. -max_concurrent_queries: 100 - -# Maximum memory usage (resident set size) for server process. -# Zero value or unset means default. Default is "max_server_memory_usage_to_ram_ratio" of available physical RAM. -# If the value is larger than "max_server_memory_usage_to_ram_ratio" of available physical RAM, it will be cut down. - -# The constraint is checked on query execution time. -# If a query tries to allocate memory and the current memory usage plus allocation is greater -# than specified threshold, exception will be thrown. - -# It is not practical to set this constraint to small values like just a few gigabytes, -# because memory allocator will keep this amount of memory in caches and the server will deny service of queries. -max_server_memory_usage: 0 - -# Maximum number of threads in the Global thread pool. -# This will default to a maximum of 10000 threads if not specified. -# This setting will be useful in scenarios where there are a large number -# of distributed queries that are running concurrently but are idling most -# of the time, in which case a higher number of threads might be required. -max_thread_pool_size: 10000 - -# On memory constrained environments you may have to set this to value larger than 1. -max_server_memory_usage_to_ram_ratio: 0.9 - -# Simple server-wide memory profiler. Collect a stack trace at every peak allocation step (in bytes). -# Data will be stored in system.trace_log table with query_id = empty string. -# Zero means disabled. -total_memory_profiler_step: 4194304 - -# Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. -# The probability is for every alloc/free regardless to the size of the allocation. -# Note that sampling happens only when the amount of untracked memory exceeds the untracked memory limit, -# which is 4 MiB by default but can be lowered if 'total_memory_profiler_step' is lowered. -# You may want to set 'total_memory_profiler_step' to 1 for extra fine grained sampling. -total_memory_tracker_sample_probability: 0 - -# Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve -# correct maximum value. -# max_open_files: 262144 - -# Size of cache of uncompressed blocks of data, used in tables of MergeTree family. -# In bytes. Cache is single for server. Memory is allocated only on demand. -# Cache is used when 'use_uncompressed_cache' user setting turned on (off by default). -# Uncompressed cache is advantageous only for very short queries and in rare cases. - -# Note: uncompressed cache can be pointless for lz4, because memory bandwidth -# is slower than multi-core decompression on some server configurations. -# Enabling it can sometimes paradoxically make queries slower. -uncompressed_cache_size: 8589934592 - -# Approximate size of mark cache, used in tables of MergeTree family. -# In bytes. Cache is single for server. Memory is allocated only on demand. -# You should not lower this value. -mark_cache_size: 5368709120 - -# If you enable the `min_bytes_to_use_mmap_io` setting, -# the data in MergeTree tables can be read with mmap to avoid copying from kernel to userspace. -# It makes sense only for large files and helps only if data reside in page cache. -# To avoid frequent open/mmap/munmap/close calls (which are very expensive due to consequent page faults) -# and to reuse mappings from several threads and queries, -# the cache of mapped files is maintained. Its size is the number of mapped regions (usually equal to the number of mapped files). -# The amount of data in mapped files can be monitored -# in system.metrics, system.metric_log by the MMappedFiles, MMappedFileBytes metrics -# and in system.asynchronous_metrics, system.asynchronous_metrics_log by the MMapCacheCells metric, -# and also in system.events, system.processes, system.query_log, system.query_thread_log, system.query_views_log by the -# CreatedReadBufferMMap, CreatedReadBufferMMapFailed, MMappedFileCacheHits, MMappedFileCacheMisses events. -# Note that the amount of data in mapped files does not consume memory directly and is not accounted -# in query or server memory usage - because this memory can be discarded similar to OS page cache. -# The cache is dropped (the files are closed) automatically on removal of old parts in MergeTree, -# also it can be dropped manually by the SYSTEM DROP MMAP CACHE query. -mmap_cache_size: 1000 - -# Cache size in bytes for compiled expressions. -compiled_expression_cache_size: 134217728 - -# Cache size in elements for compiled expressions. -compiled_expression_cache_elements_size: 10000 - -# Path to data directory, with trailing slash. -path: /var/lib/clickhouse/ - -# Path to temporary data for processing hard queries. -tmp_path: /var/lib/clickhouse/tmp/ - -# Policy from the for the temporary files. -# If not set is used, otherwise is ignored. - -# Notes: -# - move_factor is ignored -# - keep_free_space_bytes is ignored -# - max_data_part_size_bytes is ignored -# - you must have exactly one volume in that policy -# tmp_policy: tmp - -# Directory with user provided files that are accessible by 'file' table function. -user_files_path: /var/lib/clickhouse/user_files/ - -# LDAP server definitions. -ldap_servers: '' - -# List LDAP servers with their connection parameters here to later 1) use them as authenticators for dedicated local users, -# who have 'ldap' authentication mechanism specified instead of 'password', or to 2) use them as remote user directories. -# Parameters: -# host - LDAP server hostname or IP, this parameter is mandatory and cannot be empty. -# port - LDAP server port, default is 636 if enable_tls is set to true, 389 otherwise. -# bind_dn - template used to construct the DN to bind to. -# The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual -# user name during each authentication attempt. -# user_dn_detection - section with LDAP search parameters for detecting the actual user DN of the bound user. -# This is mainly used in search filters for further role mapping when the server is Active Directory. The -# resulting user DN will be used when replacing '{user_dn}' substrings wherever they are allowed. By default, -# user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected -# user DN value. -# base_dn - template used to construct the base DN for the LDAP search. -# The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings -# of the template with the actual user name and bind DN during the LDAP search. -# scope - scope of the LDAP search. -# Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). -# search_filter - template used to construct the search filter for the LDAP search. -# The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}' -# substrings of the template with the actual user name, bind DN, and base DN during the LDAP search. -# Note, that the special characters must be escaped properly in XML. -# verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed -# to be successfully authenticated for all consecutive requests without contacting the LDAP server. -# Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request. -# enable_tls - flag to trigger use of secure connection to the LDAP server. -# Specify 'no' for plain text (ldap://) protocol (not recommended). -# Specify 'yes' for LDAP over SSL/TLS (ldaps://) protocol (recommended, the default). -# Specify 'starttls' for legacy StartTLS protocol (plain text (ldap://) protocol, upgraded to TLS). -# tls_minimum_protocol_version - the minimum protocol version of SSL/TLS. -# Accepted values are: 'ssl2', 'ssl3', 'tls1.0', 'tls1.1', 'tls1.2' (the default). -# tls_require_cert - SSL/TLS peer certificate verification behavior. -# Accepted values are: 'never', 'allow', 'try', 'demand' (the default). -# tls_cert_file - path to certificate file. -# tls_key_file - path to certificate key file. -# tls_ca_cert_file - path to CA certificate file. -# tls_ca_cert_dir - path to the directory containing CA certificates. -# tls_cipher_suite - allowed cipher suite (in OpenSSL notation). -# Example: -# my_ldap_server: -# host: localhost -# port: 636 -# bind_dn: 'uid={user_name},ou=users,dc=example,dc=com' -# verification_cooldown: 300 -# enable_tls: yes -# tls_minimum_protocol_version: tls1.2 -# tls_require_cert: demand -# tls_cert_file: /path/to/tls_cert_file -# tls_key_file: /path/to/tls_key_file -# tls_ca_cert_file: /path/to/tls_ca_cert_file -# tls_ca_cert_dir: /path/to/tls_ca_cert_dir -# tls_cipher_suite: ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 - -# Example (typical Active Directory with configured user DN detection for further role mapping): -# my_ad_server: -# host: localhost -# port: 389 -# bind_dn: 'EXAMPLE\{user_name}' -# user_dn_detection: -# base_dn: CN=Users,DC=example,DC=com -# search_filter: '(&(objectClass=user)(sAMAccountName={user_name}))' -# enable_tls: no - -# To enable Kerberos authentication support for HTTP requests (GSS-SPNEGO), for those users who are explicitly configured -# to authenticate via Kerberos, define a single 'kerberos' section here. -# Parameters: -# principal - canonical service principal name, that will be acquired and used when accepting security contexts. -# This parameter is optional, if omitted, the default principal will be used. -# This parameter cannot be specified together with 'realm' parameter. -# realm - a realm, that will be used to restrict authentication to only those requests whose initiator's realm matches it. -# This parameter is optional, if omitted, no additional filtering by realm will be applied. -# This parameter cannot be specified together with 'principal' parameter. -# Example: -# kerberos: '' - -# Example: -# kerberos: -# principal: HTTP/clickhouse.example.com@EXAMPLE.COM - -# Example: -# kerberos: -# realm: EXAMPLE.COM - -# Sources to read users, roles, access rights, profiles of settings, quotas. -user_directories: - users_xml: - # Path to configuration file with predefined users. - path: users.yaml - local_directory: - # Path to folder where users created by SQL commands are stored. - path: /var/lib/clickhouse/access/ - -# # To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section -# # with the following parameters: -# # server - one of LDAP server names defined in 'ldap_servers' config section above. -# # This parameter is mandatory and cannot be empty. -# # roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. -# # If no roles are specified here or assigned during role mapping (below), user will not be able to perform any -# # actions after authentication. -# # role_mapping - section with LDAP search parameters and mapping rules. -# # When a user authenticates, while still bound to LDAP, an LDAP search is performed using search_filter and the -# # name of the logged in user. For each entry found during that search, the value of the specified attribute is -# # extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the -# # value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by -# # CREATE ROLE command. -# # There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be -# # applied. -# # base_dn - template used to construct the base DN for the LDAP search. -# # The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}' -# # substrings of the template with the actual user name, bind DN, and user DN during each LDAP search. -# # scope - scope of the LDAP search. -# # Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). -# # search_filter - template used to construct the search filter for the LDAP search. -# # The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and -# # '{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during -# # each LDAP search. -# # Note, that the special characters must be escaped properly in XML. -# # attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default. -# # prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by -# # the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated -# # as local role names. Empty, by default. -# # Example: -# # ldap: -# # server: my_ldap_server -# # roles: -# # my_local_role1: '' -# # my_local_role2: '' -# # role_mapping: -# # base_dn: 'ou=groups,dc=example,dc=com' -# # scope: subtree -# # search_filter: '(&(objectClass=groupOfNames)(member={bind_dn}))' -# # attribute: cn -# # prefix: clickhouse_ -# # Example (typical Active Directory with role mapping that relies on the detected user DN): -# # ldap: -# # server: my_ad_server -# # role_mapping: -# # base_dn: 'CN=Users,DC=example,DC=com' -# # attribute: CN -# # scope: subtree -# # search_filter: '(&(objectClass=group)(member={user_dn}))' -# # prefix: clickhouse_ - -# Default profile of settings. -default_profile: default - -# Comma-separated list of prefixes for user-defined settings. -# custom_settings_prefixes: '' -# System profile of settings. This settings are used by internal processes (Distributed DDL worker and so on). -# system_profile: default - -# Buffer profile of settings. -# This settings are used by Buffer storage to flush data to the underlying table. -# Default: used from system_profile directive. -# buffer_profile: default - -# Default database. -default_database: default - -# Server time zone could be set here. - -# Time zone is used when converting between String and DateTime types, -# when printing DateTime in text formats and parsing DateTime from text, -# it is used in date and time related functions, if specific time zone was not passed as an argument. - -# Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan. -# If not specified, system time zone at server startup is used. - -# Please note, that server could display time zone alias instead of specified name. -# Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC. -# timezone: Europe/Moscow - -# You can specify umask here (see "man umask"). Server will apply it on startup. -# Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read). -# umask: 022 - -# Perform mlockall after startup to lower first queries latency -# and to prevent clickhouse executable from being paged out under high IO load. -# Enabling this option is recommended but will lead to increased startup time for up to a few seconds. -mlock_executable: true - -# Reallocate memory for machine code ("text") using huge pages. Highly experimental. -remap_executable: false - -# Uncomment below in order to use JDBC table engine and function. -# To install and run JDBC bridge in background: -# * [Debian/Ubuntu] -# export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge -# export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') -# wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb -# apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb -# clickhouse-jdbc-bridge & -# * [CentOS/RHEL] -# export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge -# export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') -# wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm -# yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm -# clickhouse-jdbc-bridge & -# Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - -# jdbc_bridge: -# host: 127.0.0.1 -# port: 9019 - -# Configuration of clusters that could be used in Distributed tables. -# https://clickhouse.com/docs/en/operations/table_engines/distributed/ -remote_servers: - # Test only shard config for testing distributed storage - test_shard_localhost: - # Inter-server per-cluster secret for Distributed queries - # default: no secret (no authentication will be performed) - - # If set, then Distributed queries will be validated on shards, so at least: - # - such cluster should exist on the shard, - # - such cluster should have the same secret. - - # And also (and which is more important), the initial_user will - # be used as current user for the query. - - # Right now the protocol is pretty simple and it only takes into account: - # - cluster name - # - query - - # Also it will be nice if the following will be implemented: - # - source hostname (see interserver_http_host), but then it will depends from DNS, - # it can use IP address instead, but then the you need to get correct on the initiator node. - # - target hostname / ip address (same notes as for source hostname) - # - time-based security tokens - secret: 'REPLACE_ME' - shard: - # Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). - # internal_replication: false - # Optional. Shard weight when writing data. Default: 1. - # weight: 1 - replica: - host: localhost - port: 9000 - # Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). - # priority: 1 - -# The list of hosts allowed to use in URL-related storage engines and table functions. -# If this section is not present in configuration, all hosts are allowed. -# remote_url_allow_hosts: - -# Host should be specified exactly as in URL. The name is checked before DNS resolution. -# Example: "yandex.ru", "yandex.ru." and "www.yandex.ru" are different hosts. -# If port is explicitly specified in URL, the host:port is checked as a whole. -# If host specified here without port, any port with this host allowed. -# "yandex.ru" -> "yandex.ru:443", "yandex.ru:80" etc. is allowed, but "yandex.ru:80" -> only "yandex.ru:80" is allowed. -# If the host is specified as IP address, it is checked as specified in URL. Example: "[2a02:6b8:a::a]". -# If there are redirects and support for redirects is enabled, every redirect (the Location field) is checked. - -# Regular expression can be specified. RE2 engine is used for regexps. -# Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter -# (forgetting to do so is a common source of error). - -# If element has 'incl' attribute, then for it's value will be used corresponding substitution from another file. -# By default, path to file with substitutions is /etc/metrika.xml. It could be changed in config in 'include_from' element. -# Values for substitutions are specified in /clickhouse/name_of_substitution elements in that file. - -# ZooKeeper is used to store metadata about replicas, when using Replicated tables. -# Optional. If you don't use replicated tables, you could omit that. -# See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/ - -# zookeeper: -# - node: -# host: example1 -# port: 2181 -# - node: -# host: example2 -# port: 2181 -# - node: -# host: example3 -# port: 2181 - -# Substitutions for parameters of replicated tables. -# Optional. If you don't use replicated tables, you could omit that. -# See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables -# macros: -# shard: 01 -# replica: example01-01-1 - -# Reloading interval for embedded dictionaries, in seconds. Default: 3600. -builtin_dictionaries_reload_interval: 3600 - -# Maximum session timeout, in seconds. Default: 3600. -max_session_timeout: 3600 - -# Default session timeout, in seconds. Default: 60. -default_session_timeout: 60 - -# Sending data to Graphite for monitoring. Several sections can be defined. -# interval - send every X second -# root_path - prefix for keys -# hostname_in_path - append hostname to root_path (default = true) -# metrics - send data from table system.metrics -# events - send data from table system.events -# asynchronous_metrics - send data from table system.asynchronous_metrics - -# graphite: -# host: localhost -# port: 42000 -# timeout: 0.1 -# interval: 60 -# root_path: one_min -# hostname_in_path: true - -# metrics: true -# events: true -# events_cumulative: false -# asynchronous_metrics: true - -# graphite: -# host: localhost -# port: 42000 -# timeout: 0.1 -# interval: 1 -# root_path: one_sec - -# metrics: true -# events: true -# events_cumulative: false -# asynchronous_metrics: false - -# Serve endpoint for Prometheus monitoring. -# endpoint - mertics path (relative to root, statring with "/") -# port - port to setup server. If not defined or 0 than http_port used -# metrics - send data from table system.metrics -# events - send data from table system.events -# asynchronous_metrics - send data from table system.asynchronous_metrics - -# prometheus: -# endpoint: /metrics -# port: 9363 - -# metrics: true -# events: true -# asynchronous_metrics: true - -# Query log. Used only for queries with setting log_queries = 1. -query_log: - # What table to insert data. If table is not exist, it will be created. - # When query log structure is changed after system update, - # then old table will be renamed and new table will be created automatically. - database: system - table: query_log - - # PARTITION BY expr: https://clickhouse.com/docs/en/table_engines/mergetree-family/custom_partitioning_key/ - # Example: - # event_date - # toMonday(event_date) - # toYYYYMM(event_date) - # toStartOfHour(event_time) - partition_by: toYYYYMM(event_date) - - # Table TTL specification: https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl - # Example: - # event_date + INTERVAL 1 WEEK - # event_date + INTERVAL 7 DAY DELETE - # event_date + INTERVAL 2 WEEK TO DISK 'bbb' - - # ttl: 'event_date + INTERVAL 30 DAY DELETE' - - # Instead of partition_by, you can provide full engine expression (starting with ENGINE = ) with parameters, - # Example: engine: 'ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024' - - # Interval of flushing data. - flush_interval_milliseconds: 7500 - -# Trace log. Stores stack traces collected by query profilers. -# See query_profiler_real_time_period_ns and query_profiler_cpu_time_period_ns settings. -trace_log: - database: system - table: trace_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Query thread log. Has information about all threads participated in query execution. -# Used only for queries with setting log_query_threads = 1. -query_thread_log: - database: system - table: query_thread_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Query views log. Has information about all dependent views associated with a query. -# Used only for queries with setting log_query_views = 1. -query_views_log: - database: system - table: query_views_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Uncomment if use part log. -# Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads). -part_log: - database: system - table: part_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Uncomment to write text log into table. -# Text log contains all information from usual server log but stores it in structured and efficient way. -# The level of the messages that goes to the table can be limited (), if not specified all messages will go to the table. -# text_log: -# database: system -# table: text_log -# flush_interval_milliseconds: 7500 -# level: '' - -# Metric log contains rows with current values of ProfileEvents, CurrentMetrics collected with "collect_interval_milliseconds" interval. -metric_log: - database: system - table: metric_log - flush_interval_milliseconds: 7500 - collect_interval_milliseconds: 1000 - -# Asynchronous metric log contains values of metrics from -# system.asynchronous_metrics. -asynchronous_metric_log: - database: system - table: asynchronous_metric_log - - # Asynchronous metrics are updated once a minute, so there is - # no need to flush more often. - flush_interval_milliseconds: 60000 - -# OpenTelemetry log contains OpenTelemetry trace spans. -opentelemetry_span_log: - - # The default table creation code is insufficient, this spec - # is a workaround. There is no 'event_time' for this log, but two times, - # start and finish. It is sorted by finish time, to avoid inserting - # data too far away in the past (probably we can sometimes insert a span - # that is seconds earlier than the last span in the table, due to a race - # between several spans inserted in parallel). This gives the spans a - # global order that we can use to e.g. retry insertion into some external - # system. - engine: |- - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - database: system - table: opentelemetry_span_log - flush_interval_milliseconds: 7500 - -# Crash log. Stores stack traces for fatal errors. -# This table is normally empty. -crash_log: - database: system - table: crash_log - partition_by: '' - flush_interval_milliseconds: 1000 - -# Parameters for embedded dictionaries, used in Yandex.Metrica. -# See https://clickhouse.com/docs/en/dicts/internal_dicts/ - -# Path to file with region hierarchy. -# path_to_regions_hierarchy_file: /opt/geo/regions_hierarchy.txt - -# Path to directory with files containing names of regions -# path_to_regions_names_files: /opt/geo/ - - -# top_level_domains_path: /var/lib/clickhouse/top_level_domains/ -# Custom TLD lists. -# Format: name: /path/to/file - -# Changes will not be applied w/o server restart. -# Path to the list is under top_level_domains_path (see above). -top_level_domains_lists: '' - -# public_suffix_list: /path/to/public_suffix_list.dat - -# Configuration of external dictionaries. See: -# https://clickhouse.com/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts -dictionaries_config: '*_dictionary.xml' - -# Uncomment if you want data to be compressed 30-100% better. -# Don't do that if you just started using ClickHouse. - -# compression: -# # Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used. -# case: -# Conditions. All must be satisfied. Some conditions may be omitted. -# # min_part_size: 10000000000 # Min part size in bytes. -# # min_part_size_ratio: 0.01 # Min size of part relative to whole table size. -# # What compression method to use. -# method: zstd - -# Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster. -# Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -distributed_ddl: - # Path in ZooKeeper to queue with DDL queries - path: /clickhouse/task_queue/ddl - - # Settings from this profile will be used to execute DDL queries - # profile: default - - # Controls how much ON CLUSTER queries can be run simultaneously. - # pool_size: 1 - - # Cleanup settings (active tasks will not be removed) - - # Controls task TTL (default 1 week) - # task_max_lifetime: 604800 - - # Controls how often cleanup should be performed (in seconds) - # cleanup_delay_period: 60 - - # Controls how many tasks could be in the queue - # max_tasks_in_queue: 1000 - -# Settings to fine tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h -# merge_tree: -# max_suspicious_broken_parts: 5 - -# Protection from accidental DROP. -# If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query. -# If you want do delete one table and don't want to change clickhouse-server config, you could create special file /flags/force_drop_table and make DROP once. -# By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables. -# The same for max_partition_size_to_drop. -# Uncomment to disable protection. - -# max_table_size_to_drop: 0 -# max_partition_size_to_drop: 0 - -# Example of parameters for GraphiteMergeTree table engine -graphite_rollup_example: - pattern: - regexp: click_cost - function: any - retention: - - age: 0 - precision: 3600 - - age: 86400 - precision: 60 - default: - function: max - retention: - - age: 0 - precision: 60 - - age: 3600 - precision: 300 - - age: 86400 - precision: 3600 - -# Directory in containing schema files for various input formats. -# The directory will be created if it doesn't exist. -format_schema_path: /var/lib/clickhouse/format_schemas/ - -# Default query masking rules, matching lines would be replaced with something else in the logs -# (both text logs and system.query_log). -# name - name for the rule (optional) -# regexp - RE2 compatible regular expression (mandatory) -# replace - substitution string for sensitive data (optional, by default - six asterisks) -query_masking_rules: - rule: - name: hide encrypt/decrypt arguments - regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)' - # or more secure, but also more invasive: - # (aes_\w+)\s*\(.*\) - replace: \1(???) - -# Uncomment to use custom http handlers. -# rules are checked from top to bottom, first match runs the handler -# url - to match request URL, you can use 'regex:' prefix to use regex match(optional) -# methods - to match request method, you can use commas to separate multiple method matches(optional) -# headers - to match request headers, match each child element(child element name is header name), you can use 'regex:' prefix to use regex match(optional) -# handler is request handler -# type - supported types: static, dynamic_query_handler, predefined_query_handler -# query - use with predefined_query_handler type, executes query when the handler is called -# query_param_name - use with dynamic_query_handler type, extracts and executes the value corresponding to the value in HTTP request params -# status - use with static type, response status code -# content_type - use with static type, response content-type -# response_content - use with static type, Response content sent to client, when using the prefix 'file://' or 'config://', find the content from the file or configuration send to client. - -# http_handlers: -# - rule: -# url: / -# methods: POST,GET -# headers: -# pragma: no-cache -# handler: -# type: dynamic_query_handler -# query_param_name: query -# - rule: -# url: /predefined_query -# methods: POST,GET -# handler: -# type: predefined_query_handler -# query: 'SELECT * FROM system.settings' -# - rule: -# handler: -# type: static -# status: 200 -# content_type: 'text/plain; charset=UTF-8' -# response_content: config://http_server_default_response - -send_crash_reports: - # Changing to true allows sending crash reports to - # the ClickHouse core developers team via Sentry https://sentry.io - # Doing so at least in pre-production environments is highly appreciated - enabled: false - # Change to true if you don't feel comfortable attaching the server hostname to the crash report - anonymize: false - # Default endpoint should be changed to different Sentry DSN only if you have - # some in-house engineers or hired consultants who're going to debug ClickHouse issues for you - endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277' - # Uncomment to disable ClickHouse internal DNS caching. - # disable_internal_dns_cache: 1 - -storage_configuration: - disks: - s3: - secret_access_key: REPLACE_ME - access_key_id: 'REPLACE_ME' diff --git a/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml b/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml deleted file mode 100644 index c27bb7cb071..00000000000 --- a/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Users and ACL. -users: - # If user name was not specified, 'default' user is used. - default: - - password_sha256_hex: "REPLACE_ME" diff --git a/programs/diagnostics/testdata/configs/yaml/users.yaml b/programs/diagnostics/testdata/configs/yaml/users.yaml deleted file mode 100644 index 82f2d67f2a4..00000000000 --- a/programs/diagnostics/testdata/configs/yaml/users.yaml +++ /dev/null @@ -1,47 +0,0 @@ -include_from: "../include/yaml/user-include.yaml" -# Profiles of settings. -profiles: - # Default settings. - default: - # Maximum memory usage for processing single query, in bytes. - max_memory_usage: 10000000000 - load_balancing: random - - # Profile that allows only read queries. - readonly: - readonly: 1 - -# Users and ACL. -users: - # If user name was not specified, 'default' user is used. - default: - - password: 'REPLACE_ME' - - networks: - ip: '::/0' - - # Settings profile for user. - profile: default - - # Quota for user. - quota: default - - # User can create other users and grant rights to them. - # access_management: 1 - -# Quotas. -quotas: - # Name of quota. - default: - # Limits for time interval. You could specify many intervals with different limits. - interval: - # Length of interval. - duration: 3600 - - # No limits. Just calculate resource usage for time interval. - queries: 0 - errors: 0 - result_rows: 0 - read_rows: 0 - execution_time: 0 diff --git a/programs/diagnostics/testdata/configs/yandex_xml/config.xml b/programs/diagnostics/testdata/configs/yandex_xml/config.xml deleted file mode 100644 index 40d1fa34b1a..00000000000 --- a/programs/diagnostics/testdata/configs/yandex_xml/config.xml +++ /dev/null @@ -1,1167 +0,0 @@ - - - ../include/xml/server-include.xml - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - - 1000M - 10 - - - - - - - - - - - - - - - - - 8123 - - - 9000 - - - 9004 - - - 9005 - - - - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4096 - - - 3 - - - - - false - - - /path/to/ssl_cert_file - /path/to/ssl_key_file - - - false - - - /path/to/ssl_ca_cert_file - - - deflate - - - medium - - - -1 - -1 - - - false - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 100 - - - 0 - - - - 10000 - - - 0.9 - - - 4194304 - - - 0 - - - - - - 8589934592 - - - 5368709120 - - - - 1000 - - - 134217728 - - - 10000 - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - - - - /var/lib/clickhouse/user_files/ - - - - - - - - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - - - default - - - - - - - - - - - - default - - - - - - - - - true - - - false - - ' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - clickhouse-jdbc-bridge & - - * [CentOS/RHEL] - export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge - export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - clickhouse-jdbc-bridge & - - Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - ]]> - - - - - - - - - - - - - - - - localhost - 9000 - - - - - - - - - - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - - - - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - query_views_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - system - metric_log
- 7500 - 1000 - - - - - system - asynchronous_metric_log
- - 7000 - - - - - - - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - - system - opentelemetry_span_log
- 7500 - - - - - - system - crash_log
- - - 1000 - - - - - system - session_log
- - toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - - - - - *_dictionary.xml - - - *_function.xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - - hide encrypt/decrypt arguments - ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) - - \1(???) - - - - - - - - - - false - - false - - - https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 - - - - - - - - diff --git a/programs/diagnostics/testdata/docker/admin.xml b/programs/diagnostics/testdata/docker/admin.xml deleted file mode 100644 index 76aa670dcfe..00000000000 --- a/programs/diagnostics/testdata/docker/admin.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - 1 - - - - - 1 - - - \ No newline at end of file diff --git a/programs/diagnostics/testdata/docker/custom.xml b/programs/diagnostics/testdata/docker/custom.xml deleted file mode 100644 index bc1051178ca..00000000000 --- a/programs/diagnostics/testdata/docker/custom.xml +++ /dev/null @@ -1,8 +0,0 @@ - - :: - 0.0.0.0 - 1 - - 1 - - diff --git a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log deleted file mode 100644 index 1a1768fe87e..00000000000 --- a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log +++ /dev/null @@ -1,10 +0,0 @@ -2021.12.13 10:12:26.940169 [ 38398 ] {} Access(local directory): File /var/lib/clickhouse/access/users.list doesn't exist -2021.12.13 10:12:26.940204 [ 38398 ] {} Access(local directory): Recovering lists in directory /var/lib/clickhouse/access/ -2021.12.13 10:12:40.649453 [ 38445 ] {} Access(user directories): from: 127.0.0.1, user: default: Authentication failed: Code: 193. DB::Exception: Invalid credentials. (WRONG_PASSWORD), Stack trace (when copying this message, always include the lines below): - -0. DB::Exception::Exception(std::__1::basic_string, std::__1::allocator > const&, int, bool) @ 0x9b722d4 in /usr/bin/clickhouse -1. DB::IAccessStorage::throwInvalidCredentials() @ 0x119d9b27 in /usr/bin/clickhouse -2. DB::IAccessStorage::loginImpl(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&) const @ 0x119d98d7 in /usr/bin/clickhouse -3. DB::IAccessStorage::login(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&, bool) const @ 0x119d9084 in /usr/bin/clickhouse -4. DB::MultipleAccessStorage::loginImpl(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&) const @ 0x119ff93c in /usr/bin/clickhouse -5. DB::IAccessStorage::login(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&, bool) const @ 0x119d9084 in /usr/bin/clickhouse diff --git a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log deleted file mode 100644 index f6abe7764ba..00000000000 --- a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log +++ /dev/null @@ -1,10 +0,0 @@ -2022.02.02 14:49:32.458680 [ 200404 ] {} DiskLocal: Reserving 2.47 MiB on disk `default`, having unreserved 1.56 TiB. -2022.02.02 14:49:32.459086 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTask::PrepareStage: Merging 2 parts: from 202202_147058_147549_343 to 202202_147550_147550_0 into Wide -2022.02.02 14:49:32.459201 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTask::PrepareStage: Selected MergeAlgorithm: Horizontal -2022.02.02 14:49:32.459262 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTreeSequentialSource: Reading 159 marks from part 202202_147058_147549_343, total 1289014 rows starting from the beginning of the part -2022.02.02 14:49:32.459614 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTreeSequentialSource: Reading 2 marks from part 202202_147550_147550_0, total 2618 rows starting from the beginning of the part -2022.02.02 14:49:32.507755 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTask::MergeProjectionsStage: Merge sorted 1291632 rows, containing 5 columns (5 merged, 0 gathered) in 0.048711404 sec., 26516008.448452853 rows/sec., 639.52 MiB/sec. -2022.02.02 14:49:32.508332 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} system.asynchronous_metric_log (de87df8b-2250-439c-9e87-df8b2250339c): Renaming temporary part tmp_merge_202202_147058_147550_344 to 202202_147058_147550_344. -2022.02.02 14:49:32.508406 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} system.asynchronous_metric_log (de87df8b-2250-439c-9e87-df8b2250339c) (MergerMutator): Merged 2 parts: from 202202_147058_147549_343 to 202202_147550_147550_0 -2022.02.02 14:49:32.508440 [ 200359 ] {} MemoryTracker: Peak memory usage Mutate/Merge: 16.31 MiB. -2022.02.02 14:49:33.000148 [ 200388 ] {} AsynchronousMetrics: MemoryTracking: was 774.16 MiB, peak 2.51 GiB, will set to 772.30 MiB (RSS), difference: -1.86 MiB diff --git a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz deleted file mode 100644 index 136bf5913aa..00000000000 --- a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz +++ /dev/null @@ -1 +0,0 @@ -dummy hz file for tests diff --git a/programs/disks/DisksApp.cpp b/programs/disks/DisksApp.cpp index b7c3c7f5c97..6c768799221 100644 --- a/programs/disks/DisksApp.cpp +++ b/programs/disks/DisksApp.cpp @@ -166,7 +166,7 @@ int DisksApp::main(const std::vector & /*args*/) { String config_path = config().getString("config-file", getDefaultConfigFileName()); ConfigProcessor config_processor(config_path, false, false); - config_processor.setConfigPath(fs::path(config_path).parent_path()); + ConfigProcessor::setConfigPath(fs::path(config_path).parent_path()); auto loaded_config = config_processor.loadConfig(); config().add(loaded_config.configuration.duplicate(), false, false); } diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 50f801f2560..d4b975ce1e8 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -102,7 +103,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) { std::string_view name = field.getName(); if (name == "max_parser_depth" || name == "max_query_size") - cmd_settings.addProgramOption(desc, name, field); + addProgramOption(cmd_settings, desc, name, field); } boost::program_options::variables_map options; @@ -236,7 +237,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ASTPtr res = parseQueryAndMovePosition( parser, pos, end, "query", multiple, cmd_settings.max_query_size, cmd_settings.max_parser_depth, cmd_settings.max_parser_backtracks); - std::unique_ptr insert_query_payload = nullptr; + std::unique_ptr insert_query_payload; /// If the query is INSERT ... VALUES, then we will try to parse the data. if (auto * insert_query = res->as(); insert_query && insert_query->data) { diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index 0ff0faff624..20c1a0ad4a8 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -662,7 +662,6 @@ int mainEntryClickHouseInstall(int argc, char ** argv) " \n" " " << (config_dir / "server.crt").string() << "\n" " " << (config_dir / "server.key").string() << "\n" - " " << (config_dir / "dhparam.pem").string() << "\n" " \n" " \n" "\n"; diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp index 8297fab5ed9..52d825f30e6 100644 --- a/programs/keeper-client/KeeperClient.cpp +++ b/programs/keeper-client/KeeperClient.cpp @@ -368,7 +368,7 @@ int KeeperClient::main(const std::vector & /* args */) DB::ConfigProcessor config_processor(config().getString("config-file", "config.xml")); /// This will handle a situation when clickhouse is running on the embedded config, but config.d folder is also present. - config_processor.registerEmbeddedConfig("config.xml", ""); + ConfigProcessor::registerEmbeddedConfig("config.xml", ""); auto clickhouse_config = config_processor.loadConfig(); Poco::Util::AbstractConfiguration::Keys keys; diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt index 98d8848502d..2fca10ce4d7 100644 --- a/programs/library-bridge/CMakeLists.txt +++ b/programs/library-bridge/CMakeLists.txt @@ -24,9 +24,4 @@ target_link_libraries(clickhouse-library-bridge PRIVATE set_target_properties(clickhouse-library-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) -if (SPLIT_DEBUG_SYMBOLS) - clickhouse_split_debug_symbols(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-library-bridge) -else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) - install(TARGETS clickhouse-library-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) -endif() +install(TARGETS clickhouse-library-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) diff --git a/programs/library-bridge/ExternalDictionaryLibraryUtils.h b/programs/library-bridge/ExternalDictionaryLibraryUtils.h index c9d03d27f75..e6bf8f2a4c3 100644 --- a/programs/library-bridge/ExternalDictionaryLibraryUtils.h +++ b/programs/library-bridge/ExternalDictionaryLibraryUtils.h @@ -35,7 +35,7 @@ public: ExternalDictionaryLibraryAPI::CStrings strings; // will pass pointer to lib private: - std::unique_ptr ptr_holder = nullptr; + std::unique_ptr ptr_holder; Container strings_holder; }; diff --git a/programs/local/CMakeLists.txt b/programs/local/CMakeLists.txt index 565b67d0020..1aaa2859898 100644 --- a/programs/local/CMakeLists.txt +++ b/programs/local/CMakeLists.txt @@ -25,9 +25,3 @@ endif() # Always use internal readpassphrase target_link_libraries(clickhouse-local-lib PRIVATE readpassphrase) - -if (ENABLE_FUZZING) - add_compile_definitions(FUZZING_MODE=1) - set (WITH_COVERAGE ON) - target_link_libraries(clickhouse-local-lib PRIVATE ${LIB_FUZZING_ENGINE}) -endif () diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 167d361f5e3..72920fbd855 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -22,8 +21,6 @@ #include #include #include -#include -#include #include #include #include @@ -34,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -42,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -61,10 +56,6 @@ #include "config.h" -#if defined(FUZZING_MODE) - #include -#endif - #if USE_AZURE_BLOB_STORAGE # include #endif @@ -131,7 +122,7 @@ void LocalServer::initialize(Poco::Util::Application & self) { const auto config_path = config().getString("config-file", "config.xml"); ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(fs::path(config_path).parent_path()); + ConfigProcessor::setConfigPath(fs::path(config_path).parent_path()); auto loaded_config = config_processor.loadConfig(); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } @@ -321,47 +312,28 @@ void LocalServer::cleanup() } -static bool checkIfStdinIsRegularFile() -{ - struct stat file_stat; - return fstat(STDIN_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode); -} - - -static bool checkIfStdoutIsRegularFile() -{ - struct stat file_stat; - return fstat(STDOUT_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode); -} - - std::string LocalServer::getInitialCreateTableQuery() { - if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!checkIfStdinIsRegularFile() || queries.empty())) + if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!isRegularFile(STDIN_FILENO) || queries.empty())) return {}; auto table_name = backQuoteIfNeed(config().getString("table-name", "table")); auto table_structure = config().getString("table-structure", "auto"); String table_file; - std::optional format_from_file_name; if (!config().has("table-file") || config().getString("table-file") == "-") { /// Use Unix tools stdin naming convention table_file = "stdin"; - format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDIN_FILENO); } else { /// Use regular file auto file_name = config().getString("table-file"); table_file = quoteString(file_name); - format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(file_name); } - auto data_format = backQuoteIfNeed( - config().getString("table-data-format", config().getString("format", format_from_file_name ? *format_from_file_name : "TSV"))); - + String data_format = backQuoteIfNeed(default_input_format); if (table_structure == "auto") table_structure = ""; @@ -474,25 +446,10 @@ try } } -#if defined(FUZZING_MODE) - static bool first_time = true; - if (first_time) - { - - if (queries_files.empty() && queries.empty()) - { - std::cerr << "\033[31m" << "ClickHouse compiled in fuzzing mode." << "\033[0m" << std::endl; - std::cerr << "\033[31m" << "You have to provide a query with --query or --queries-file option." << "\033[0m" << std::endl; - std::cerr << "\033[31m" << "The query have to use function getFuzzerData() inside." << "\033[0m" << std::endl; - exit(1); - } - - is_interactive = false; -#else is_interactive = stdin_is_a_tty && (config().hasOption("interactive") || (queries.empty() && !config().has("table-structure") && queries_files.empty() && !config().has("table-file"))); -#endif + if (!is_interactive) { /// We will terminate process on error @@ -538,15 +495,13 @@ try connect(); -#ifdef FUZZING_MODE - first_time = false; - } -#endif - String initial_query = getInitialCreateTableQuery(); if (!initial_query.empty()) processQueryText(initial_query); +#if defined(FUZZING_MODE) + runLibFuzzer(); +#else if (is_interactive && !delayed_interactive) { runInteractive(); @@ -558,10 +513,8 @@ try if (delayed_interactive) runInteractive(); } - -#ifndef FUZZING_MODE - cleanup(); #endif + return Application::EXIT_OK; } catch (const DB::Exception & e) @@ -646,26 +599,7 @@ void LocalServer::processConfig() if (config().has("macros")) global_context->setMacros(std::make_unique(config(), "macros", log)); - if (!config().has("output-format") && !config().has("format") && checkIfStdoutIsRegularFile()) - { - std::optional format_from_file_name; - format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDOUT_FILENO); - format = format_from_file_name ? *format_from_file_name : "TSV"; - } - else - format = config().getString("output-format", config().getString("format", is_interactive ? "PrettyCompact" : "TSV")); - insert_format = "Values"; - - /// Setting value from cmd arg overrides one from config - if (global_context->getSettingsRef().max_insert_block_size.changed) - { - insert_format_max_block_size = global_context->getSettingsRef().max_insert_block_size; - } - else - { - insert_format_max_block_size = config().getUInt64("insert_format_max_block_size", - global_context->getSettingsRef().max_insert_block_size); - } + setDefaultFormatsFromConfiguration(); /// Sets external authenticators config (LDAP, Kerberos). global_context->setExternalAuthenticatorsConfig(config()); @@ -829,23 +763,11 @@ void LocalServer::processConfig() void LocalServer::printHelpMessage([[maybe_unused]] const OptionsDescription & options_description) { -#if defined(FUZZING_MODE) - std::cout << - "usage: clickhouse -- \n" - "Note: It is important not to use only one letter keys with single dash for \n" - "for clickhouse-local arguments. It may work incorrectly.\n" - - "ClickHouse is build with coverage guided fuzzer (libfuzzer) inside it.\n" - "You have to provide a query which contains getFuzzerData function.\n" - "This will take the data from fuzzing engine, pass it to getFuzzerData function and execute a query.\n" - "Each time the data will be different, and it will last until some segfault or sanitizer assertion is found. \n"; -#else std::cout << getHelpHeader() << "\n"; std::cout << options_description.main_description.value() << "\n"; std::cout << getHelpFooter() << "\n"; std::cout << "In addition, --param_name=value can be specified for substitution of parameters for parametrized queries.\n"; std::cout << "\nSee also: https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/\n"; -#endif } @@ -859,7 +781,6 @@ void LocalServer::addOptions(OptionsDescription & options_description) ("file,F", po::value(), "path to file with data of the initial table (stdin if not specified)") ("input-format", po::value(), "input format of the initial table data") - ("output-format", po::value(), "default output format") ("logger.console", po::value()->implicit_value(true), "Log to console") ("logger.log", po::value(), "Log file name") @@ -921,6 +842,7 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum for (int arg_num = 1; arg_num < argc; ++arg_num) { std::string_view arg = argv[arg_num]; + /// Parameter arg after underline. if (arg.starts_with("--param_")) { @@ -953,7 +875,9 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum addMultiquery(arg, common_arguments); } else + { common_arguments.emplace_back(arg); + } } } @@ -988,67 +912,3 @@ int mainEntryClickHouseLocal(int argc, char ** argv) return code ? code : 1; } } - -#if defined(FUZZING_MODE) - -// linked from programs/main.cpp -bool isClickhouseApp(const std::string & app_suffix, std::vector & argv); - -std::optional fuzz_app; - -extern "C" int LLVMFuzzerInitialize(int * pargc, char *** pargv) -{ - std::vector argv(*pargv, *pargv + (*pargc + 1)); - - /// As a user you can add flags to clickhouse binary in fuzzing mode as follows - /// clickhouse local -- - - char **p = &(*pargv)[1]; - - auto it = argv.begin() + 1; - for (; *it; ++it) - { - if (strcmp(*it, "--") == 0) - { - ++it; - break; - } - } - - while (*it) - { - if (strncmp(*it, "--", 2) != 0) - { - *(p++) = *it; - it = argv.erase(it); - } - else - ++it; - } - - *pargc = static_cast(p - &(*pargv)[0]); - *p = nullptr; - - /// Initialize clickhouse-local app - fuzz_app.emplace(); - fuzz_app->init(static_cast(argv.size() - 1), argv.data()); - - return 0; -} - - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size) -{ - try - { - auto input = String(reinterpret_cast(data), size); - DB::FunctionGetFuzzerData::update(input); - fuzz_app->run(); - } - catch (...) - { - } - - return 0; -} -#endif diff --git a/programs/main.cpp b/programs/main.cpp index 0a35594bd30..9ad8b016c82 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -67,7 +67,6 @@ namespace using MainFunc = int (*)(int, char**); -#if !defined(FUZZING_MODE) /// Add an item here to register new application std::pair clickhouse_applications[] = { @@ -111,7 +110,6 @@ int printHelp(int, char **) std::cerr << "clickhouse " << application.first << " [args] " << std::endl; return -1; } -#endif /// Add an item here to register a new short name std::pair clickhouse_short_names[] = @@ -284,7 +282,7 @@ struct Checker ; -#if !defined(FUZZING_MODE) && !defined(USE_MUSL) +#if !defined(USE_MUSL) /// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete. void checkHarmfulEnvironmentVariables(char ** argv) { @@ -446,13 +444,8 @@ extern "C" /// /// extern bool inside_main; /// class C { C() { assert(inside_main); } }; -#ifndef FUZZING_MODE bool inside_main = false; -#else -bool inside_main = true; -#endif -#if !defined(FUZZING_MODE) int main(int argc_, char ** argv_) { inside_main = true; @@ -494,7 +487,7 @@ int main(int argc_, char ** argv_) /// Interpret binary without argument or with arguments starts with dash /// ('-') as clickhouse-local for better usability: /// - /// clickhouse # dumps help + /// clickhouse help # dumps help /// clickhouse -q 'select 1' # use local /// clickhouse # spawn local /// clickhouse local # spawn local @@ -510,4 +503,3 @@ int main(int argc_, char ** argv_) return exit_code; } -#endif diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index 18cda4d7a04..83839cc21ac 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -4,8 +4,8 @@ set (CLICKHOUSE_ODBC_BRIDGE_SOURCES ColumnInfoHandler.cpp IdentifierQuoteHandler.cpp MainHandler.cpp - ODBCBlockInputStream.cpp - ODBCBlockOutputStream.cpp + ODBCSource.cpp + ODBCSink.cpp ODBCBridge.cpp ODBCHandlerFactory.cpp PingHandler.cpp @@ -30,12 +30,7 @@ target_link_libraries(clickhouse-odbc-bridge PRIVATE set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) target_compile_options (clickhouse-odbc-bridge PRIVATE -Wno-reserved-id-macro -Wno-keyword-macro) -if (SPLIT_DEBUG_SYMBOLS) - clickhouse_split_debug_symbols(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-odbc-bridge) -else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) - install(TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) -endif() +install(TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) if(ENABLE_TESTS) add_subdirectory(tests) diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 4cb15de3b2c..5ff985b3d12 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/programs/odbc-bridge/ColumnInfoHandler.h b/programs/odbc-bridge/ColumnInfoHandler.h index ca7044fdf32..610fb128c9d 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.h +++ b/programs/odbc-bridge/ColumnInfoHandler.h @@ -5,7 +5,6 @@ #if USE_ODBC #include -#include #include #include diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index e350afa2b10..2cf1576ccd7 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -1,8 +1,8 @@ #include "MainHandler.h" #include "validateODBCConnectionString.h" -#include "ODBCBlockInputStream.h" -#include "ODBCBlockOutputStream.h" +#include "ODBCSource.h" +#include "ODBCSink.h" #include "getIdentifierQuote.h" #include #include diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCSink.cpp similarity index 97% rename from programs/odbc-bridge/ODBCBlockOutputStream.cpp rename to programs/odbc-bridge/ODBCSink.cpp index 37b70023169..ea2e88690ce 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.cpp +++ b/programs/odbc-bridge/ODBCSink.cpp @@ -1,4 +1,4 @@ -#include "ODBCBlockOutputStream.h" +#include "ODBCSink.h" #include #include diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.h b/programs/odbc-bridge/ODBCSink.h similarity index 100% rename from programs/odbc-bridge/ODBCBlockOutputStream.h rename to programs/odbc-bridge/ODBCSink.h diff --git a/programs/odbc-bridge/ODBCBlockInputStream.cpp b/programs/odbc-bridge/ODBCSource.cpp similarity index 98% rename from programs/odbc-bridge/ODBCBlockInputStream.cpp rename to programs/odbc-bridge/ODBCSource.cpp index 59a5deac960..7f0d47f7e2e 100644 --- a/programs/odbc-bridge/ODBCBlockInputStream.cpp +++ b/programs/odbc-bridge/ODBCSource.cpp @@ -1,11 +1,10 @@ -#include "ODBCBlockInputStream.h" +#include "ODBCSource.h" #include #include #include #include #include #include -#include namespace DB diff --git a/programs/odbc-bridge/ODBCBlockInputStream.h b/programs/odbc-bridge/ODBCSource.h similarity index 100% rename from programs/odbc-bridge/ODBCBlockInputStream.h rename to programs/odbc-bridge/ODBCSource.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b67a4eccd15..a048bebc45b 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -733,13 +734,17 @@ try LOG_INFO(log, "Available CPU instruction sets: {}", cpu_info); #endif + bool will_have_trace_collector = hasPHDRCache() && config().has("trace_log"); + // Initialize global thread pool. Do it before we fetch configs from zookeeper // nodes (`from_zk`), because ZooKeeper interface uses the pool. We will // ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well. GlobalThreadPool::initialize( server_settings.max_thread_pool_size, server_settings.max_thread_pool_free_size, - server_settings.thread_pool_queue_size); + server_settings.thread_pool_queue_size, + will_have_trace_collector ? server_settings.global_profiler_real_time_period_ns : 0, + will_have_trace_collector ? server_settings.global_profiler_cpu_time_period_ns : 0); /// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed). SCOPE_EXIT({ Stopwatch watch; @@ -1439,6 +1444,7 @@ try global_context->getProcessList().setMaxSize(new_server_settings.max_concurrent_queries); global_context->getProcessList().setMaxInsertQueriesAmount(new_server_settings.max_concurrent_insert_queries); global_context->getProcessList().setMaxSelectQueriesAmount(new_server_settings.max_concurrent_select_queries); + global_context->getProcessList().setMaxWaitingQueriesAmount(new_server_settings.max_waiting_queries); if (config->has("keeper_server")) global_context->updateKeeperConfiguration(*config); diff --git a/programs/server/config.xml b/programs/server/config.xml index ea3ead47c32..e92381eeb1e 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -96,7 +96,7 @@ https://{bucket}.s3.amazonaws.com - https://{bucket}.storage.googleapis.com + https://storage.googleapis.com/{bucket} https://{bucket}.oss.aliyuncs.com diff --git a/rust/CMakeLists.txt b/rust/CMakeLists.txt index 6715a54221a..cebfd36a24a 100644 --- a/rust/CMakeLists.txt +++ b/rust/CMakeLists.txt @@ -99,6 +99,19 @@ function(add_rust_subdirectory src) message(STATUS "Copy ${src} to ${dst}") file(COPY "${src}" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}" PATTERN target EXCLUDE) + + # Check is Rust available or not. + # + # `cargo update --dry-run` will not update anything, but will check the internet connectivity. + execute_process(COMMAND ${Rust_CARGO_CACHED} update --dry-run + WORKING_DIRECTORY "${dst}" + RESULT_VARIABLE CARGO_UPDATE_RESULT + OUTPUT_VARIABLE CARGO_UPDATE_STDOUT + ERROR_VARIABLE CARGO_UPDATE_STDERR) + if (CARGO_UPDATE_RESULT) + message(FATAL_ERROR "Rust (${Rust_CARGO_CACHED}) support is not available (likely there is no internet connectivity):\n${CARGO_UPDATE_STDERR}\nYou can disable Rust support with -DENABLE_RUST=OFF") + endif() + add_subdirectory("${dst}" "${dst}") # cmake -E copy* do now know how to exclude files diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index ba89899dd8f..1110b9c4b21 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -16,6 +16,8 @@ #include #include #include + +#include #include namespace fs = std::filesystem; diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 72cbeca4f11..a87e9361e8e 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -1,6 +1,8 @@ #include -#include #include +#include +#include + #include #include #include @@ -408,6 +410,65 @@ public: friend bool operator!=(const Node & left, const Node & right) { return !(left == right); } + bool contains(const Node & other) + { + if (min_flags_with_children.contains(other.max_flags_with_children)) + return true; + + if (!flags.contains(other.flags)) + return false; + + /// Let's assume that the current node has the following rights: + /// + /// SELECT ON *.* TO user1; + /// REVOKE SELECT ON system.* FROM user1; + /// REVOKE SELECT ON mydb.* FROM user1; + /// + /// And the other node has the rights: + /// + /// SELECT ON *.* TO user2; + /// REVOKE SELECT ON system.* FROM user2; + /// + /// First, we check that each child from the other node is present in the current node: + /// + /// SELECT ON *.* TO user1; -- checked + /// REVOKE SELECT ON system.* FROM user1; -- checked + if (other.children) + { + for (const auto & [name, node] : *other.children) + { + const auto & child = tryGetChild(name); + if (child == nullptr) + { + if (!flags.contains(node.flags)) + return false; + } + else + { + if (!child->contains(node)) + return false; + } + } + } + + if (!children) + return true; + + /// Then we check that each of our children has no other rights revoked. + /// + /// REVOKE SELECT ON mydb.* FROM user1; -- check failed, returning false + for (const auto & [name, node] : *children) + { + if (other.children && other.children->contains(name)) + continue; + + if (!node.flags.contains(other.flags)) + return false; + } + + return true; + } + void makeUnion(const Node & other) { makeUnionRec(other); @@ -1004,6 +1065,24 @@ bool AccessRights::isGrantedImpl(const AccessFlags & flags, const Args &... args return helper(root); } +template +bool AccessRights::containsImpl(const AccessRights & other) const +{ + auto helper = [&](const std::unique_ptr & root_node) -> bool + { + if (!root_node) + return !other.root; + if (!other.root) + return true; + return root_node->contains(*other.root); + }; + if constexpr (grant_option) + return helper(root_with_grant_option); + else + return helper(root); +} + + template bool AccessRights::isGrantedImplHelper(const AccessRightsElement & element) const { @@ -1068,6 +1147,8 @@ bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view da bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl(element); } bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl(elements); } +bool AccessRights::contains(const AccessRights & access_rights) const { return containsImpl(access_rights); } +bool AccessRights::containsWithGrantOption(const AccessRights & access_rights) const { return containsImpl(access_rights); } bool operator ==(const AccessRights & left, const AccessRights & right) { diff --git a/src/Access/AccessRights.h b/src/Access/AccessRights.h index 5efffc0037a..bfb4b7c68c3 100644 --- a/src/Access/AccessRights.h +++ b/src/Access/AccessRights.h @@ -95,6 +95,10 @@ public: bool hasGrantOption(const AccessRightsElement & element) const; bool hasGrantOption(const AccessRightsElements & elements) const; + /// Checks if a given `access_rights` is a subset for the current access rights. + bool contains(const AccessRights & access_rights) const; + bool containsWithGrantOption(const AccessRights & access_rights) const; + /// Merges two sets of access rights together. /// It's used to combine access rights from multiple roles. void makeUnion(const AccessRights & other); @@ -153,6 +157,9 @@ private: template bool isGrantedImpl(const AccessRightsElements & elements) const; + template + bool containsImpl(const AccessRights & other) const; + template bool isGrantedImplHelper(const AccessRightsElement & element) const; diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 47187d83154..bf1fe3feec3 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -4,11 +4,12 @@ #include #include #include -#include #include +#include +#include #include -#include +#include "config.h" namespace DB { @@ -74,7 +75,7 @@ namespace } #if USE_SSH - bool checkSshSignature(const std::vector & keys, std::string_view signature, std::string_view original) + bool checkSshSignature(const std::vector & keys, std::string_view signature, std::string_view original) { for (const auto & key: keys) if (key.isPublic() && key.verifySignature(signature, original)) @@ -114,7 +115,11 @@ bool Authentication::areCredentialsValid( throw Authentication::Require("ClickHouse X.509 Authentication"); case AuthenticationType::SSH_KEY: - throw Authentication::Require("Ssh Keys Authentication"); +#if USE_SSH + throw Authentication::Require("SSH Keys Authentication"); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); +#endif case AuthenticationType::MAX: break; @@ -145,7 +150,11 @@ bool Authentication::areCredentialsValid( throw Authentication::Require("ClickHouse X.509 Authentication"); case AuthenticationType::SSH_KEY: - throw Authentication::Require("Ssh Keys Authentication"); +#if USE_SSH + throw Authentication::Require("SSH Keys Authentication"); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); +#endif case AuthenticationType::MAX: break; @@ -178,7 +187,11 @@ bool Authentication::areCredentialsValid( throw Authentication::Require("ClickHouse X.509 Authentication"); case AuthenticationType::SSH_KEY: - throw Authentication::Require("Ssh Keys Authentication"); +#if USE_SSH + throw Authentication::Require("SSH Keys Authentication"); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); +#endif case AuthenticationType::BCRYPT_PASSWORD: return checkPasswordBcrypt(basic_credentials->getPassword(), auth_data.getPasswordHashBinary()); @@ -216,13 +229,18 @@ bool Authentication::areCredentialsValid( return auth_data.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName()); case AuthenticationType::SSH_KEY: - throw Authentication::Require("Ssh Keys Authentication"); +#if USE_SSH + throw Authentication::Require("SSH Keys Authentication"); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); +#endif case AuthenticationType::MAX: break; } } +#if USE_SSH if (const auto * ssh_credentials = typeid_cast(&credentials)) { switch (auth_data.getType()) @@ -243,15 +261,12 @@ bool Authentication::areCredentialsValid( throw Authentication::Require("ClickHouse X.509 Authentication"); case AuthenticationType::SSH_KEY: -#if USE_SSH return checkSshSignature(auth_data.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal()); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL"); -#endif case AuthenticationType::MAX: break; } } +#endif if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp index da90a0f5842..a4c25b438e8 100644 --- a/src/Access/AuthenticationData.cpp +++ b/src/Access/AuthenticationData.cpp @@ -105,7 +105,10 @@ bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs) return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash) && (lhs.ldap_server_name == rhs.ldap_server_name) && (lhs.kerberos_realm == rhs.kerberos_realm) && (lhs.ssl_certificate_common_names == rhs.ssl_certificate_common_names) - && (lhs.ssh_keys == rhs.ssh_keys) && (lhs.http_auth_scheme == rhs.http_auth_scheme) +#if USE_SSH + && (lhs.ssh_keys == rhs.ssh_keys) +#endif + && (lhs.http_auth_scheme == rhs.http_auth_scheme) && (lhs.http_auth_server_name == rhs.http_auth_server_name); } @@ -326,7 +329,7 @@ std::shared_ptr AuthenticationData::toAST() const break; #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); #endif } case AuthenticationType::HTTP: @@ -355,7 +358,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que { #if USE_SSH AuthenticationData auth_data(*query.type); - std::vector keys; + std::vector keys; size_t args_size = query.children.size(); for (size_t i = 0; i < args_size; ++i) @@ -366,7 +369,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que try { - keys.emplace_back(ssh::SSHKeyFactory::makePublicFromBase64(key_base64, type)); + keys.emplace_back(SSHKeyFactory::makePublicKeyFromBase64(key_base64, type)); } catch (const std::invalid_argument &) { @@ -377,7 +380,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que auth_data.setSSHKeys(std::move(keys)); return auth_data; #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); #endif } diff --git a/src/Access/AuthenticationData.h b/src/Access/AuthenticationData.h index feef4d71d66..c97e0327b56 100644 --- a/src/Access/AuthenticationData.h +++ b/src/Access/AuthenticationData.h @@ -2,14 +2,16 @@ #include #include +#include #include #include -#include #include #include #include +#include "config.h" + namespace DB { @@ -59,8 +61,10 @@ public: const boost::container::flat_set & getSSLCertificateCommonNames() const { return ssl_certificate_common_names; } void setSSLCertificateCommonNames(boost::container::flat_set common_names_); - const std::vector & getSSHKeys() const { return ssh_keys; } - void setSSHKeys(std::vector && ssh_keys_) { ssh_keys = std::forward>(ssh_keys_); } +#if USE_SSH + const std::vector & getSSHKeys() const { return ssh_keys; } + void setSSHKeys(std::vector && ssh_keys_) { ssh_keys = std::forward>(ssh_keys_); } +#endif HTTPAuthenticationScheme getHTTPAuthenticationScheme() const { return http_auth_scheme; } void setHTTPAuthenticationScheme(HTTPAuthenticationScheme scheme) { http_auth_scheme = scheme; } @@ -94,7 +98,9 @@ private: String kerberos_realm; boost::container::flat_set ssl_certificate_common_names; String salt; - std::vector ssh_keys; +#if USE_SSH + std::vector ssh_keys; +#endif /// HTTP authentication properties String http_auth_server_name; HTTPAuthenticationScheme http_auth_scheme = HTTPAuthenticationScheme::BASIC; diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 6095f8ce6d3..83b50ce96c3 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -205,7 +205,7 @@ enum class AccessType M(SYSTEM_FLUSH, "", GROUP, SYSTEM) \ M(SYSTEM_THREAD_FUZZER, "SYSTEM START THREAD FUZZER, SYSTEM STOP THREAD FUZZER, START THREAD FUZZER, STOP THREAD FUZZER", GLOBAL, SYSTEM) \ M(SYSTEM_UNFREEZE, "SYSTEM UNFREEZE", GLOBAL, SYSTEM) \ - M(SYSTEM_FAILPOINT, "SYSTEM ENABLE FAILPOINT, SYSTEM DISABLE FAILPOINT", GLOBAL, SYSTEM) \ + M(SYSTEM_FAILPOINT, "SYSTEM ENABLE FAILPOINT, SYSTEM DISABLE FAILPOINT, SYSTEM WAIT FAILPOINT", GLOBAL, SYSTEM) \ M(SYSTEM_LISTEN, "SYSTEM START LISTEN, SYSTEM STOP LISTEN", GLOBAL, SYSTEM) \ M(SYSTEM_JEMALLOC, "SYSTEM JEMALLOC PURGE, SYSTEM JEMALLOC ENABLE PROFILE, SYSTEM JEMALLOC DISABLE PROFILE, SYSTEM JEMALLOC FLUSH PROFILE", GLOBAL, SYSTEM) \ M(SYSTEM, "", GROUP, ALL) /* allows to execute SYSTEM {SHUTDOWN|RELOAD CONFIG|...} */ \ diff --git a/src/Access/Common/AuthenticationType.h b/src/Access/Common/AuthenticationType.h index 48ace3ca00a..506c8abd3b1 100644 --- a/src/Access/Common/AuthenticationType.h +++ b/src/Access/Common/AuthenticationType.h @@ -34,8 +34,8 @@ enum class AuthenticationType /// Password is encrypted in bcrypt hash. BCRYPT_PASSWORD, - /// Server sends a random string named `challenge` which client needs to encrypt with private key. - /// The check is performed on server side by decrypting the data and comparing with the original string. + /// Server sends a random string named `challenge` to the client. The client encrypts it with its SSH private key. + /// The server decrypts the result using the SSH public key registered for the user and compares with the original string. SSH_KEY, /// Authentication through HTTP protocol diff --git a/src/Access/Common/QuotaDefs.cpp b/src/Access/Common/QuotaDefs.cpp index 04c16a562d2..2a22b80f65d 100644 --- a/src/Access/Common/QuotaDefs.cpp +++ b/src/Access/Common/QuotaDefs.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index 77b90eaaebc..d04f8a66541 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -3,6 +3,7 @@ #include #include +#include "config.h" namespace DB { @@ -86,10 +87,11 @@ class MySQLNative41Credentials : public CredentialsWithScramble using CredentialsWithScramble::CredentialsWithScramble; }; +#if USE_SSH class SshCredentials : public Credentials { public: - explicit SshCredentials(const String& user_name_, const String& signature_, const String& original_) + SshCredentials(const String & user_name_, const String & signature_, const String & original_) : Credentials(user_name_), signature(signature_), original(original_) { is_ready = true; @@ -117,5 +119,6 @@ private: String signature; String original; }; +#endif } diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index ad78bf92e02..b00c8bac849 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -13,6 +13,8 @@ #include #include +#include + namespace Poco { class Logger; } namespace Poco::Net { class IPAddress; } diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 91477e5f3aa..ef5cf722113 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -30,7 +31,7 @@ void User::setName(const String & name_) throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name is empty"); if (name_ == EncodedUserInfo::USER_INTERSERVER_MARKER) throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_); - if (startsWith(name_, EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER)) + if (name_.starts_with(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_); name = name_; } diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index b4b843fc77e..e3c45eb45ae 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -10,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -214,7 +214,7 @@ namespace Poco::Util::AbstractConfiguration::Keys entries; config.keys(ssh_keys_config, entries); - std::vector keys; + std::vector keys; for (const String& entry : entries) { const auto conf_pref = ssh_keys_config + "." + entry + "."; @@ -237,7 +237,7 @@ namespace try { - keys.emplace_back(ssh::SSHKeyFactory::makePublicFromBase64(base64_key, type)); + keys.emplace_back(SSHKeyFactory::makePublicKeyFromBase64(base64_key, type)); } catch (const std::invalid_argument &) { @@ -249,7 +249,7 @@ namespace } user->auth_data.setSSHKeys(std::move(keys)); #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); #endif } else if (has_http_auth) diff --git a/src/AggregateFunctions/AggregateFunctionAnyHeavy.cpp b/src/AggregateFunctions/AggregateFunctionAnyHeavy.cpp index 4f4d4a19cba..ffddd46f2e3 100644 --- a/src/AggregateFunctions/AggregateFunctionAnyHeavy.cpp +++ b/src/AggregateFunctions/AggregateFunctionAnyHeavy.cpp @@ -115,34 +115,34 @@ public: void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { - this->data(place).add(*columns[0], row_num, arena); + data(place).add(*columns[0], row_num, arena); } void addManyDefaults(AggregateDataPtr __restrict place, const IColumn ** columns, size_t, Arena * arena) const override { - this->data(place).addManyDefaults(*columns[0], 0, arena); + data(place).addManyDefaults(*columns[0], 0, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - this->data(place).add(this->data(rhs), arena); + data(place).add(data(rhs), arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf, *serialization); + data(place).write(buf, *serialization); } void deserialize(AggregateDataPtr place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, *serialization, arena); + data(place).read(buf, *serialization, arena); } bool allocatesMemoryInArena() const override { return singleValueTypeAllocatesMemoryInArena(value_type_index); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - this->data(place).insertResultInto(to); + data(place).insertResultInto(to); } }; diff --git a/src/AggregateFunctions/AggregateFunctionCount.cpp b/src/AggregateFunctions/AggregateFunctionCount.cpp index ec91685dbe6..25f991ab693 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.cpp +++ b/src/AggregateFunctions/AggregateFunctionCount.cpp @@ -5,6 +5,12 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + struct Settings; AggregateFunctionPtr AggregateFunctionCount::getOwnNullAdapter( @@ -19,7 +25,9 @@ namespace AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { assertNoParameters(name, parameters); - assertArityAtMost<1>(name, argument_types); + + if (argument_types.size() > 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires zero or one argument", name); return std::make_shared(argument_types); } diff --git a/src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp b/src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp index 8582c8c56fc..3d7d6eff608 100644 --- a/src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp +++ b/src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp @@ -54,30 +54,30 @@ public: { const auto & value = columns[0]->getFloat64(row_num); const auto & time = columns[1]->getFloat64(row_num); - this->data(place).add(value, time, half_decay); + data(place).add(value, time, half_decay); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override { - this->data(place).merge(this->data(rhs), half_decay); + data(place).merge(data(rhs), half_decay); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - writeBinary(this->data(place).value, buf); - writeBinary(this->data(place).time, buf); + writeBinary(data(place).value, buf); + writeBinary(data(place).time, buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override { - readBinary(this->data(place).value, buf); - readBinary(this->data(place).time, buf); + readBinary(data(place).value, buf); + readBinary(data(place).time, buf); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { auto & column = assert_cast &>(to); - column.getData().push_back(this->data(place).get(half_decay)); + column.getData().push_back(data(place).get(half_decay)); } }; diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index 18edb7c8ce0..6555ae63128 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -1,11 +1,11 @@ #include #include - #include #include #include #include #include +#include static constexpr size_t MAX_AGGREGATE_FUNCTION_NAME_LENGTH = 1000; diff --git a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp index f3d99046036..33e318b6c2f 100644 --- a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp +++ b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp @@ -559,7 +559,7 @@ public: ptr = ptrs[row_num]; } - this->data(place).add(ptr, allocated, trace_values.data() + prev_offset, trace_size, arena); + data(place).add(ptr, allocated, trace_values.data() + prev_offset, trace_size, arena); } void addManyDefaults( @@ -572,7 +572,7 @@ public: void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - this->data(place).merge(this->data(rhs), arena); + data(place).merge(data(rhs), arena); } void serialize(ConstAggregateDataPtr __restrict, WriteBuffer &, std::optional /* version */) const override @@ -590,7 +590,7 @@ public: auto & array = assert_cast(to); auto & str = assert_cast(array.getData()); - this->data(place).dumpFlameGraph(str.getChars(), str.getOffsets(), 0, 0); + data(place).dumpFlameGraph(str.getChars(), str.getOffsets(), 0, 0); array.getOffsets().push_back(str.size()); } diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 6af8b1018dd..63002652166 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -89,10 +89,10 @@ struct GroupArraySamplerData chassert(lim != 0); /// With a large number of values, we will generate random numbers several times slower. - if (lim <= static_cast(rng.max())) + if (lim <= static_cast(pcg32_fast::max())) return rng() % lim; else - return (static_cast(rng()) * (static_cast(rng.max()) + 1ULL) + static_cast(rng())) % lim; + return (static_cast(rng()) * (static_cast(pcg32::max()) + 1ULL) + static_cast(rng())) % lim; } void randomShuffle() diff --git a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp index 882150325be..e1224fae2fb 100644 --- a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp @@ -293,32 +293,32 @@ public: Float64 value = columns[0]->getFloat64(row_num); UInt8 is_second = columns[1]->getUInt(row_num); if (is_second) - this->data(place).addY(value, arena); + data(place).addY(value, arena); else - this->data(place).addX(value, arena); + data(place).addX(value, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - this->data(place).merge(this->data(rhs), arena); + data(place).merge(data(rhs), arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf); + data(place).write(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, arena); + data(place).read(buf, arena); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - if (!this->data(place).size_x || !this->data(place).size_y) + if (!data(place).size_x || !data(place).size_y) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} require both samples to be non empty", getName()); - auto [d_statistic, p_value] = this->data(place).getResult(alternative, method); + auto [d_statistic, p_value] = data(place).getResult(alternative, method); /// Because p-value is a probability. p_value = std::min(1.0, std::max(0.0, p_value)); diff --git a/src/AggregateFunctions/AggregateFunctionLargestTriangleThreeBuckets.cpp b/src/AggregateFunctions/AggregateFunctionLargestTriangleThreeBuckets.cpp index d5abdbc12fb..b24b6c8996f 100644 --- a/src/AggregateFunctions/AggregateFunctionLargestTriangleThreeBuckets.cpp +++ b/src/AggregateFunctions/AggregateFunctionLargestTriangleThreeBuckets.cpp @@ -242,7 +242,7 @@ public: { Float64 x = getFloat64DataFromColumn(columns[0], row_num, this->x_type); Float64 y = getFloat64DataFromColumn(columns[1], row_num, this->y_type); - this->data(place).add(x, y, arena); + data(place).add(x, y, arena); } Float64 getFloat64DataFromColumn(const IColumn * column, size_t row_num, TypeIndex type_index) const @@ -264,25 +264,25 @@ public: void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - auto & a = this->data(place); - const auto & b = this->data(rhs); + auto & a = data(place); + const auto & b = data(rhs); a.merge(b, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf); + data(place).write(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, arena); + data(place).read(buf, arena); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const override { - auto res = this->data(place).getResult(total_buckets, arena); + auto res = data(place).getResult(total_buckets, arena); auto & col = assert_cast(to); auto & col_offsets = assert_cast(col.getOffsetsColumn()); diff --git a/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp b/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp index a70da7b35d5..e7bc5df335f 100644 --- a/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp +++ b/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp @@ -205,35 +205,35 @@ public: UInt8 is_second = columns[1]->getUInt(row_num); if (is_second) - this->data(place).addY(value, arena); + data(place).addY(value, arena); else - this->data(place).addX(value, arena); + data(place).addX(value, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override { - auto & a = this->data(place); - const auto & b = this->data(rhs); + auto & a = data(place); + const auto & b = data(rhs); a.merge(b, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf); + data(place).write(buf); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override { - this->data(place).read(buf, arena); + data(place).read(buf, arena); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - if (!this->data(place).size_x || !this->data(place).size_y) + if (!data(place).size_x || !data(place).size_y) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} require both samples to be non empty", getName()); - auto [u_statistic, p_value] = this->data(place).getResult(alternative, continuity_correction); + auto [u_statistic, p_value] = data(place).getResult(alternative, continuity_correction); /// Because p-value is a probability. p_value = std::min(1.0, std::max(0.0, p_value)); diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.cpp b/src/AggregateFunctions/AggregateFunctionQuantile.cpp index 110f6c6b4d6..4d37ec69d26 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.cpp +++ b/src/AggregateFunctions/AggregateFunctionQuantile.cpp @@ -14,6 +14,7 @@ struct Settings; namespace ErrorCodes { + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NOT_IMPLEMENTED; } @@ -116,8 +117,8 @@ template