mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-27 01:51:59 +00:00
Merge branch 'master' into optimize_uniq_to_count2
This commit is contained in:
JackyWoo
commit
c4ae83bddc
16
.clang-tidy
16
.clang-tidy
@ -5,6 +5,9 @@
|
||||
# a) the new check is not controversial (this includes many checks in readability-* and google-*) or
|
||||
# b) too noisy (checks with > 100 new warnings are considered noisy, this includes e.g. cppcoreguidelines-*).
|
||||
|
||||
# TODO: Once clang(-tidy) 17 is the minimum, we can convert this list to YAML
|
||||
# See https://releases.llvm.org/17.0.1/tools/clang/tools/extra/docs/ReleaseNotes.html#improvements-to-clang-tidy
|
||||
|
||||
# TODO Let clang-tidy check headers in further directories
|
||||
# --> HeaderFilterRegex: '^.*/(src|base|programs|utils)/.*(h|hpp)$'
|
||||
HeaderFilterRegex: '^.*/(base)/.*(h|hpp)$'
|
||||
@ -25,6 +28,7 @@ Checks: '*,
|
||||
-bugprone-not-null-terminated-result,
|
||||
-bugprone-reserved-identifier, # useful but too slow, TODO retry when https://reviews.llvm.org/rG1c282052624f9d0bd273bde0b47b30c96699c6c7 is merged
|
||||
-bugprone-unchecked-optional-access,
|
||||
-bugprone-*, -- category temporarily disabled because some check(s) in it are slow
|
||||
|
||||
-cert-dcl16-c,
|
||||
-cert-dcl37-c,
|
||||
@ -39,6 +43,7 @@ Checks: '*,
|
||||
-clang-analyzer-optin.portability.UnixAPI,
|
||||
-clang-analyzer-security.insecureAPI.bzero,
|
||||
-clang-analyzer-security.insecureAPI.strcpy,
|
||||
-clang-analyzer-*, -- category temporarily disabled because some check(s) in it are slow
|
||||
|
||||
-cppcoreguidelines-avoid-c-arrays,
|
||||
-cppcoreguidelines-avoid-const-or-ref-data-members,
|
||||
@ -67,6 +72,7 @@ Checks: '*,
|
||||
-cppcoreguidelines-pro-type-vararg,
|
||||
-cppcoreguidelines-slicing,
|
||||
-cppcoreguidelines-special-member-functions,
|
||||
-cppcoreguidelines-*, -- category temporarily disabled because some check(s) in it are slow
|
||||
|
||||
-darwin-*,
|
||||
|
||||
@ -108,6 +114,7 @@ Checks: '*,
|
||||
-openmp-*,
|
||||
|
||||
-misc-const-correctness,
|
||||
-misc-include-cleaner, # useful but far too many occurrences
|
||||
-misc-no-recursion,
|
||||
-misc-non-private-member-variables-in-classes,
|
||||
-misc-confusable-identifiers, # useful but slooow
|
||||
@ -127,10 +134,12 @@ Checks: '*,
|
||||
|
||||
-performance-inefficient-string-concatenation,
|
||||
-performance-no-int-to-ptr,
|
||||
-performance-avoid-endl,
|
||||
-performance-unnecessary-value-param,
|
||||
|
||||
-portability-simd-intrinsics,
|
||||
|
||||
-readability-avoid-unconditional-preprocessor-if,
|
||||
-readability-braces-around-statements,
|
||||
-readability-convert-member-functions-to-static,
|
||||
-readability-else-after-return,
|
||||
@ -154,6 +163,13 @@ Checks: '*,
|
||||
|
||||
WarningsAsErrors: '*'
|
||||
|
||||
ExtraArgs:
|
||||
# clang-tidy 17 started to complain (for unknown reasons) that various pragmas are unknown ("clang-diagnostic-unknown-pragmas").
|
||||
# This is technically a compiler error, not a clang-tidy error. We could litter the code base with more pragmas that suppress
|
||||
# this error but it is better to pass the following flag to the compiler:
|
||||
- '-Wno-unknown-pragmas'
|
||||
- '-Wno-unused-command-line-argument' # similar issue
|
||||
|
||||
CheckOptions:
|
||||
readability-identifier-naming.ClassCase: CamelCase
|
||||
readability-identifier-naming.EnumCase: CamelCase
|
||||
|
||||
4
.github/workflows/docs_check.yml
vendored
4
.github/workflows/docs_check.yml
vendored
@ -13,9 +13,7 @@ on: # yamllint disable-line rule:truthy
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- 'CHANGELOG.md'
|
||||
- 'README.md'
|
||||
- 'SECURITY.md'
|
||||
- '**.md'
|
||||
- 'docker/docs/**'
|
||||
- 'docs/**'
|
||||
- 'utils/check-style/aspell-ignore/**'
|
||||
|
||||
4
.github/workflows/nightly.yml
vendored
4
.github/workflows/nightly.yml
vendored
@ -82,8 +82,8 @@ jobs:
|
||||
SONAR_SCANNER_VERSION: 4.8.0.2856
|
||||
SONAR_SERVER_URL: "https://sonarcloud.io"
|
||||
BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
|
||||
CC: clang-16
|
||||
CXX: clang++-16
|
||||
CC: clang-17
|
||||
CXX: clang++-17
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
|
||||
4
.github/workflows/pull_request.yml
vendored
4
.github/workflows/pull_request.yml
vendored
@ -13,9 +13,7 @@ on: # yamllint disable-line rule:truthy
|
||||
branches:
|
||||
- master
|
||||
paths-ignore:
|
||||
- 'CHANGELOG.md'
|
||||
- 'README.md'
|
||||
- 'SECURITY.md'
|
||||
- '**.md'
|
||||
- 'docker/docs/**'
|
||||
- 'docs/**'
|
||||
- 'utils/check-style/aspell-ignore/**'
|
||||
|
||||
18
CHANGELOG.md
18
CHANGELOG.md
@ -25,13 +25,13 @@
|
||||
* Add column `ptr` to `system.trace_log` for `trace_type = 'MemorySample'`. This column contains an address of allocation. Added function `flameGraph` which can build flamegraph containing allocated and not released memory. Reworking of [#38391](https://github.com/ClickHouse/ClickHouse/issues/38391). [#45322](https://github.com/ClickHouse/ClickHouse/pull/45322) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Added table function `azureBlobStorageCluster`. The supported set of features is very similar to table function `s3Cluster`. [#50795](https://github.com/ClickHouse/ClickHouse/pull/50795) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
|
||||
* Allow using `cluster`, `clusterAllReplicas`, `remote`, and `remoteSecure` without table name in issue [#50808](https://github.com/ClickHouse/ClickHouse/issues/50808). [#50848](https://github.com/ClickHouse/ClickHouse/pull/50848) ([Yangkuan Liu](https://github.com/LiuYangkuan)).
|
||||
* A system table to monitor kafka consumers. [#50999](https://github.com/ClickHouse/ClickHouse/pull/50999) ([Ilya Golshtein](https://github.com/ilejn)).
|
||||
* A system table to monitor Kafka consumers. [#50999](https://github.com/ClickHouse/ClickHouse/pull/50999) ([Ilya Golshtein](https://github.com/ilejn)).
|
||||
* Added `max_sessions_for_user` setting. [#51724](https://github.com/ClickHouse/ClickHouse/pull/51724) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
|
||||
* New functions `toUTCTimestamp/fromUTCTimestamp` to act same as spark's `to_utc_timestamp/from_utc_timestamp`. [#52117](https://github.com/ClickHouse/ClickHouse/pull/52117) ([KevinyhZou](https://github.com/KevinyhZou)).
|
||||
* Add new functions `structureToCapnProtoSchema`/`structureToProtobufSchema` that convert ClickHouse table structure to CapnProto/Protobuf format schema. Allow to input/output data in CapnProto/Protobuf format without external format schema using autogenerated schema from table structure (controled by settings `format_capn_proto_use_autogenerated_schema`/`format_protobuf_use_autogenerated_schema`). Allow to export autogenerated schema while input/outoput using setting `output_format_schema`. [#52278](https://github.com/ClickHouse/ClickHouse/pull/52278) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Add new functions `structureToCapnProtoSchema`/`structureToProtobufSchema` that convert ClickHouse table structure to CapnProto/Protobuf format schema. Allow to input/output data in CapnProto/Protobuf format without external format schema using autogenerated schema from table structure (controlled by settings `format_capn_proto_use_autogenerated_schema`/`format_protobuf_use_autogenerated_schema`). Allow to export autogenerated schema while input/output using setting `output_format_schema`. [#52278](https://github.com/ClickHouse/ClickHouse/pull/52278) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* A new field `query_cache_usage` in `system.query_log` now shows if and how the query cache was used. [#52384](https://github.com/ClickHouse/ClickHouse/pull/52384) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add new function `startsWithUTF8` and `endsWithUTF8`. [#52555](https://github.com/ClickHouse/ClickHouse/pull/52555) ([李扬](https://github.com/taiyang-li)).
|
||||
* Allow variable number of columns in TSV/CuatomSeprarated/JSONCompactEachRow, make schema inference work with variable number of columns. Add settings `input_format_tsv_allow_variable_number_of_columns`, `input_format_custom_allow_variable_number_of_columns`, `input_format_json_compact_allow_variable_number_of_columns`. [#52692](https://github.com/ClickHouse/ClickHouse/pull/52692) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Allow variable number of columns in TSV/CustomSeparated/JSONCompactEachRow, make schema inference work with variable number of columns. Add settings `input_format_tsv_allow_variable_number_of_columns`, `input_format_custom_allow_variable_number_of_columns`, `input_format_json_compact_allow_variable_number_of_columns`. [#52692](https://github.com/ClickHouse/ClickHouse/pull/52692) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Added `SYSTEM STOP/START PULLING REPLICATION LOG` queries (for testing `ReplicatedMergeTree`). [#52881](https://github.com/ClickHouse/ClickHouse/pull/52881) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Allow to execute constant non-deterministic functions in mutations on initiator. [#53129](https://github.com/ClickHouse/ClickHouse/pull/53129) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Add input format `One` that doesn't read any data and always returns single row with column `dummy` with type `UInt8` and value `0` like `system.one`. It can be used together with `_file/_path` virtual columns to list files in file/s3/url/hdfs/etc table functions without reading any data. [#53209](https://github.com/ClickHouse/ClickHouse/pull/53209) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
@ -55,7 +55,7 @@
|
||||
* Parquet filter pushdown. I.e. when reading Parquet files, row groups (chunks of the file) are skipped based on the WHERE condition and the min/max values in each column. In particular, if the file is roughly sorted by some column, queries that filter by a short range of that column will be much faster. [#52951](https://github.com/ClickHouse/ClickHouse/pull/52951) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Optimize reading small row groups by batching them together in Parquet. Closes [#53069](https://github.com/ClickHouse/ClickHouse/issues/53069). [#53281](https://github.com/ClickHouse/ClickHouse/pull/53281) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Optimize count from files in most input formats. Closes [#44334](https://github.com/ClickHouse/ClickHouse/issues/44334). [#53637](https://github.com/ClickHouse/ClickHouse/pull/53637) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Use filter by file/path before reading in `url`/`file`/`hdfs` table functins. [#53529](https://github.com/ClickHouse/ClickHouse/pull/53529) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Use filter by file/path before reading in `url`/`file`/`hdfs` table functions. [#53529](https://github.com/ClickHouse/ClickHouse/pull/53529) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Enable JIT compilation for AArch64, PowerPC, SystemZ, RISC-V. [#38217](https://github.com/ClickHouse/ClickHouse/pull/38217) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Add setting `rewrite_count_distinct_if_with_count_distinct_implementation` to rewrite `countDistinctIf` with `count_distinct_implementation`. Closes [#30642](https://github.com/ClickHouse/ClickHouse/issues/30642). [#46051](https://github.com/ClickHouse/ClickHouse/pull/46051) ([flynn](https://github.com/ucasfl)).
|
||||
* Speed up merging of states of `uniq` and `uniqExact` aggregate functions by parallelizing conversion before merge. [#50748](https://github.com/ClickHouse/ClickHouse/pull/50748) ([Jiebin Sun](https://github.com/jiebinn)).
|
||||
@ -535,7 +535,7 @@
|
||||
* Add MemoryTracker for the background tasks (merges and mutation). Introduces `merges_mutations_memory_usage_soft_limit` and `merges_mutations_memory_usage_to_ram_ratio` settings that represent the soft memory limit for merges and mutations. If this limit is reached ClickHouse won't schedule new merge or mutation tasks. Also `MergesMutationsMemoryTracking` metric is introduced to allow observing current memory usage of background tasks. Resubmit [#46089](https://github.com/ClickHouse/ClickHouse/issues/46089). Closes [#48774](https://github.com/ClickHouse/ClickHouse/issues/48774). [#48787](https://github.com/ClickHouse/ClickHouse/pull/48787) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Function `dotProduct` work for array. [#49050](https://github.com/ClickHouse/ClickHouse/pull/49050) ([FFFFFFFHHHHHHH](https://github.com/FFFFFFFHHHHHHH)).
|
||||
* Support statement `SHOW INDEX` to improve compatibility with MySQL. [#49158](https://github.com/ClickHouse/ClickHouse/pull/49158) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add virtual column `_file` and `_path` support to table function `url`. - Impove error message for table function `url`. - resolves [#49231](https://github.com/ClickHouse/ClickHouse/issues/49231) - resolves [#49232](https://github.com/ClickHouse/ClickHouse/issues/49232). [#49356](https://github.com/ClickHouse/ClickHouse/pull/49356) ([Ziyi Tan](https://github.com/Ziy1-Tan)).
|
||||
* Add virtual column `_file` and `_path` support to table function `url`. - Improve error message for table function `url`. - resolves [#49231](https://github.com/ClickHouse/ClickHouse/issues/49231) - resolves [#49232](https://github.com/ClickHouse/ClickHouse/issues/49232). [#49356](https://github.com/ClickHouse/ClickHouse/pull/49356) ([Ziyi Tan](https://github.com/Ziy1-Tan)).
|
||||
* Adding the `grants` field in the users.xml file, which allows specifying grants for users. [#49381](https://github.com/ClickHouse/ClickHouse/pull/49381) ([pufit](https://github.com/pufit)).
|
||||
* Support full/right join by using grace hash join algorithm. [#49483](https://github.com/ClickHouse/ClickHouse/pull/49483) ([lgbo](https://github.com/lgbo-ustc)).
|
||||
* `WITH FILL` modifier groups filling by sorting prefix. Controlled by `use_with_fill_by_sorting_prefix` setting (enabled by default). Related to [#33203](https://github.com/ClickHouse/ClickHouse/issues/33203)#issuecomment-1418736794. [#49503](https://github.com/ClickHouse/ClickHouse/pull/49503) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
@ -580,7 +580,7 @@
|
||||
* `DEFLATE_QPL` codec lower the minimum simd version to SSE 4.2. [doc change in qpl](https://github.com/intel/qpl/commit/3f8f5cea27739f5261e8fd577dc233ffe88bf679) - Intel® QPL relies on a run-time kernels dispatcher and cpuid check to choose the best available implementation(sse/avx2/avx512) - restructured cmakefile for qpl build in clickhouse to align with latest upstream qpl. [#49811](https://github.com/ClickHouse/ClickHouse/pull/49811) ([jasperzhu](https://github.com/jinjunzh)).
|
||||
* Add initial support to do JOINs with pure parallel replicas. [#49544](https://github.com/ClickHouse/ClickHouse/pull/49544) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* More parallelism on `Outdated` parts removal with "zero-copy replication". [#49630](https://github.com/ClickHouse/ClickHouse/pull/49630) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Parallel Replicas: 1) Fixed an error `NOT_FOUND_COLUMN_IN_BLOCK` in case of using parallel replicas with non-replicated storage with disabled setting `parallel_replicas_for_non_replicated_merge_tree` 2) Now `allow_experimental_parallel_reading_from_replicas` have 3 possible values - 0, 1 and 2. 0 - disabled, 1 - enabled, silently disable them in case of failure (in case of FINAL or JOIN), 2 - enabled, throw an expection in case of failure. 3) If FINAL modifier is used in SELECT query and parallel replicas are enabled, ClickHouse will try to disable them if `allow_experimental_parallel_reading_from_replicas` is set to 1 and throw an exception otherwise. [#50195](https://github.com/ClickHouse/ClickHouse/pull/50195) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Parallel Replicas: 1) Fixed an error `NOT_FOUND_COLUMN_IN_BLOCK` in case of using parallel replicas with non-replicated storage with disabled setting `parallel_replicas_for_non_replicated_merge_tree` 2) Now `allow_experimental_parallel_reading_from_replicas` have 3 possible values - 0, 1 and 2. 0 - disabled, 1 - enabled, silently disable them in case of failure (in case of FINAL or JOIN), 2 - enabled, throw an exception in case of failure. 3) If FINAL modifier is used in SELECT query and parallel replicas are enabled, ClickHouse will try to disable them if `allow_experimental_parallel_reading_from_replicas` is set to 1 and throw an exception otherwise. [#50195](https://github.com/ClickHouse/ClickHouse/pull/50195) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* When parallel replicas are enabled they will always skip unavailable servers (the behavior is controlled by the setting `skip_unavailable_shards`, enabled by default and can be only disabled). This closes: [#48565](https://github.com/ClickHouse/ClickHouse/issues/48565). [#50293](https://github.com/ClickHouse/ClickHouse/pull/50293) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
|
||||
#### Improvement
|
||||
@ -599,7 +599,7 @@
|
||||
* Add a new column `zookeeper_name` in system.replicas, to indicate on which (auxiliary) zookeeper cluster the replicated table's metadata is stored. [#48549](https://github.com/ClickHouse/ClickHouse/pull/48549) ([cangyin](https://github.com/cangyin)).
|
||||
* `IN` operator support the comparison of `Date` and `Date32`. Closes [#48736](https://github.com/ClickHouse/ClickHouse/issues/48736). [#48806](https://github.com/ClickHouse/ClickHouse/pull/48806) ([flynn](https://github.com/ucasfl)).
|
||||
* Support for erasure codes in `HDFS`, author: @M1eyu2018, @tomscut. [#48833](https://github.com/ClickHouse/ClickHouse/pull/48833) ([M1eyu](https://github.com/M1eyu2018)).
|
||||
* Implement SYSTEM DROP REPLICA from auxillary ZooKeeper clusters, may be close [#48931](https://github.com/ClickHouse/ClickHouse/issues/48931). [#48932](https://github.com/ClickHouse/ClickHouse/pull/48932) ([wangxiaobo](https://github.com/wzb5212)).
|
||||
* Implement SYSTEM DROP REPLICA from auxiliary ZooKeeper clusters, may be close [#48931](https://github.com/ClickHouse/ClickHouse/issues/48931). [#48932](https://github.com/ClickHouse/ClickHouse/pull/48932) ([wangxiaobo](https://github.com/wzb5212)).
|
||||
* Add Array data type to MongoDB. Closes [#48598](https://github.com/ClickHouse/ClickHouse/issues/48598). [#48983](https://github.com/ClickHouse/ClickHouse/pull/48983) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Support storing `Interval` data types in tables. [#49085](https://github.com/ClickHouse/ClickHouse/pull/49085) ([larryluogit](https://github.com/larryluogit)).
|
||||
* Allow using `ntile` window function without explicit window frame definition: `ntile(3) OVER (ORDER BY a)`, close [#46763](https://github.com/ClickHouse/ClickHouse/issues/46763). [#49093](https://github.com/ClickHouse/ClickHouse/pull/49093) ([vdimir](https://github.com/vdimir)).
|
||||
@ -655,7 +655,7 @@
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* New and improved `keeper-bench`. Everything can be customized from YAML/XML file: - request generator - each type of request generator can have a specific set of fields - multi requests can be generated just by doing the same under `multi` key - for each request or subrequest in multi a `weight` field can be defined to control distribution - define trees that need to be setup for a test run - hosts can be defined with all timeouts customizable and it's possible to control how many sessions to generate for each host - integers defined with `min_value` and `max_value` fields are random number generators. [#48547](https://github.com/ClickHouse/ClickHouse/pull/48547) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Io_uring is not supported on macos, don't choose it when running tests on local to avoid occassional failures. [#49250](https://github.com/ClickHouse/ClickHouse/pull/49250) ([Frank Chen](https://github.com/FrankChen021)).
|
||||
* Io_uring is not supported on macos, don't choose it when running tests on local to avoid occasional failures. [#49250](https://github.com/ClickHouse/ClickHouse/pull/49250) ([Frank Chen](https://github.com/FrankChen021)).
|
||||
* Support named fault injection for testing. [#49361](https://github.com/ClickHouse/ClickHouse/pull/49361) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Allow running ClickHouse in the OS where the `prctl` (process control) syscall is not available, such as AWS Lambda. [#49538](https://github.com/ClickHouse/ClickHouse/pull/49538) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fixed the issue of build conflict between contrib/isa-l and isa-l in qpl [49296](https://github.com/ClickHouse/ClickHouse/issues/49296). [#49584](https://github.com/ClickHouse/ClickHouse/pull/49584) ([jasperzhu](https://github.com/jinjunzh)).
|
||||
@ -1359,7 +1359,7 @@ Add settings input_format_tsv/csv/custom_detect_header that enable this behaviou
|
||||
* Use already written part of the query for fuzzy search (pass to the `skim` library, which is written in Rust and linked statically to ClickHouse). [#44600](https://github.com/ClickHouse/ClickHouse/pull/44600) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Enable `input_format_json_read_objects_as_strings` by default to be able to read nested JSON objects while JSON Object type is experimental. [#44657](https://github.com/ClickHouse/ClickHouse/pull/44657) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Improvement for deduplication of async inserts: when users do duplicate async inserts, we should deduplicate inside the memory before we query Keeper. [#44682](https://github.com/ClickHouse/ClickHouse/pull/44682) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Input/ouptut `Avro` format will parse bool type as ClickHouse bool type. [#44684](https://github.com/ClickHouse/ClickHouse/pull/44684) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Input/output `Avro` format will parse bool type as ClickHouse bool type. [#44684](https://github.com/ClickHouse/ClickHouse/pull/44684) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Support Bool type in Arrow/Parquet/ORC. Closes [#43970](https://github.com/ClickHouse/ClickHouse/issues/43970). [#44698](https://github.com/ClickHouse/ClickHouse/pull/44698) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Don't greedily parse beyond the quotes when reading UUIDs - it may lead to mistakenly successful parsing of incorrect data. [#44686](https://github.com/ClickHouse/ClickHouse/pull/44686) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Infer UInt64 in case of Int64 overflow and fix some transforms in schema inference. [#44696](https://github.com/ClickHouse/ClickHouse/pull/44696) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -102,17 +102,6 @@ if (ENABLE_FUZZING)
|
||||
set (ENABLE_PROTOBUF 1)
|
||||
endif()
|
||||
|
||||
option (ENABLE_WOBOQ_CODEBROWSER "Build for woboq codebrowser" OFF)
|
||||
|
||||
if (ENABLE_WOBOQ_CODEBROWSER)
|
||||
set (ENABLE_EMBEDDED_COMPILER 0)
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-poison-system-directories")
|
||||
# woboq codebrowser uses clang tooling, and they could add default system
|
||||
# clang includes, and later clang will warn for those added by itself
|
||||
# includes.
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-poison-system-directories")
|
||||
endif()
|
||||
|
||||
# Global libraries
|
||||
# See:
|
||||
# - default_libs.cmake
|
||||
|
||||
@ -16,7 +16,6 @@ curl https://clickhouse.com/ | sh
|
||||
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
|
||||
* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
|
||||
* [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events.
|
||||
* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlighting and navigation.
|
||||
* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev.
|
||||
* [Static Analysis (SonarCloud)](https://sonarcloud.io/project/issues?resolved=false&id=ClickHouse_ClickHouse) proposes C++ quality improvements.
|
||||
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
|
||||
|
||||
@ -177,7 +177,7 @@ inline bool memequalWide(const char * p1, const char * p2, size_t size)
|
||||
return false;
|
||||
}
|
||||
|
||||
switch (size / 16)
|
||||
switch (size / 16) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case 3: if (!compare8(p1 + 32, p2 + 32)) return false; [[fallthrough]];
|
||||
case 2: if (!compare8(p1 + 16, p2 + 16)) return false; [[fallthrough]];
|
||||
|
||||
@ -185,6 +185,6 @@
|
||||
|
||||
/// A template function for suppressing warnings about unused variables or function results.
|
||||
template <typename... Args>
|
||||
constexpr void UNUSED(Args &&... args [[maybe_unused]])
|
||||
constexpr void UNUSED(Args &&... args [[maybe_unused]]) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
}
|
||||
|
||||
@ -20,14 +20,14 @@ Out & dumpValue(Out &, T &&);
|
||||
|
||||
/// Catch-all case.
|
||||
template <int priority, typename Out, typename T>
|
||||
std::enable_if_t<priority == -1, Out> & dumpImpl(Out & out, T &&)
|
||||
std::enable_if_t<priority == -1, Out> & dumpImpl(Out & out, T &&) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
return out << "{...}";
|
||||
}
|
||||
|
||||
/// An object, that could be output with operator <<.
|
||||
template <int priority, typename Out, typename T>
|
||||
std::enable_if_t<priority == 0, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::declval<Out &>() << std::declval<T>())> * = nullptr)
|
||||
std::enable_if_t<priority == 0, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::declval<Out &>() << std::declval<T>())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
return out << x;
|
||||
}
|
||||
@ -37,7 +37,7 @@ template <int priority, typename Out, typename T>
|
||||
std::enable_if_t<priority == 1
|
||||
/// Protect from the case when operator * do effectively nothing (function pointer).
|
||||
&& !std::is_same_v<std::decay_t<T>, std::decay_t<decltype(*std::declval<T>())>>
|
||||
, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(*std::declval<T>())> * = nullptr)
|
||||
, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(*std::declval<T>())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
if (!x)
|
||||
return out << "nullptr";
|
||||
@ -46,7 +46,7 @@ std::enable_if_t<priority == 1
|
||||
|
||||
/// Container.
|
||||
template <int priority, typename Out, typename T>
|
||||
std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::begin(std::declval<T>()))> * = nullptr)
|
||||
std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::begin(std::declval<T>()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
bool first = true;
|
||||
out << "{";
|
||||
@ -64,7 +64,7 @@ std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<
|
||||
|
||||
template <int priority, typename Out, typename T>
|
||||
std::enable_if_t<priority == 3 && std::is_enum_v<std::decay_t<T>>, Out> &
|
||||
dumpImpl(Out & out, T && x)
|
||||
dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
return out << magic_enum::enum_name(x);
|
||||
}
|
||||
@ -73,7 +73,7 @@ dumpImpl(Out & out, T && x)
|
||||
|
||||
template <int priority, typename Out, typename T>
|
||||
std::enable_if_t<priority == 3 && (std::is_same_v<std::decay_t<T>, std::string> || std::is_same_v<std::decay_t<T>, const char *>), Out> &
|
||||
dumpImpl(Out & out, T && x)
|
||||
dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
return out << std::quoted(x);
|
||||
}
|
||||
@ -82,7 +82,7 @@ dumpImpl(Out & out, T && x)
|
||||
|
||||
template <int priority, typename Out, typename T>
|
||||
std::enable_if_t<priority == 3 && std::is_same_v<std::decay_t<T>, unsigned char>, Out> &
|
||||
dumpImpl(Out & out, T && x)
|
||||
dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
return out << int(x);
|
||||
}
|
||||
@ -90,7 +90,7 @@ dumpImpl(Out & out, T && x)
|
||||
|
||||
/// Tuple, pair
|
||||
template <size_t N, typename Out, typename T>
|
||||
Out & dumpTupleImpl(Out & out, T && x)
|
||||
Out & dumpTupleImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
if constexpr (N == 0)
|
||||
out << "{";
|
||||
@ -108,14 +108,14 @@ Out & dumpTupleImpl(Out & out, T && x)
|
||||
}
|
||||
|
||||
template <int priority, typename Out, typename T>
|
||||
std::enable_if_t<priority == 4, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::get<0>(std::declval<T>()))> * = nullptr)
|
||||
std::enable_if_t<priority == 4, Out> & dumpImpl(Out & out, T && x, std::decay_t<decltype(std::get<0>(std::declval<T>()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
return dumpTupleImpl<0>(out, x);
|
||||
}
|
||||
|
||||
|
||||
template <int priority, typename Out, typename T>
|
||||
Out & dumpDispatchPriorities(Out & out, T && x, std::decay_t<decltype(dumpImpl<priority>(std::declval<Out &>(), std::declval<T>()))> *)
|
||||
Out & dumpDispatchPriorities(Out & out, T && x, std::decay_t<decltype(dumpImpl<priority>(std::declval<Out &>(), std::declval<T>()))> *) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
return dumpImpl<priority>(out, x);
|
||||
}
|
||||
@ -124,21 +124,21 @@ Out & dumpDispatchPriorities(Out & out, T && x, std::decay_t<decltype(dumpImpl<p
|
||||
struct LowPriority { LowPriority(void *) {} };
|
||||
|
||||
template <int priority, typename Out, typename T>
|
||||
Out & dumpDispatchPriorities(Out & out, T && x, LowPriority)
|
||||
Out & dumpDispatchPriorities(Out & out, T && x, LowPriority) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
return dumpDispatchPriorities<priority - 1>(out, x, nullptr);
|
||||
}
|
||||
|
||||
|
||||
template <typename Out, typename T>
|
||||
Out & dumpValue(Out & out, T && x)
|
||||
Out & dumpValue(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
return dumpDispatchPriorities<5>(out, x, nullptr);
|
||||
}
|
||||
|
||||
|
||||
template <typename Out, typename T>
|
||||
Out & dump(Out & out, const char * name, T && x)
|
||||
Out & dump(Out & out, const char * name, T && x) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
// Dumping string literal, printing name and demangled type is irrelevant.
|
||||
if constexpr (std::is_same_v<const char *, std::decay_t<std::remove_reference_t<T>>>)
|
||||
|
||||
@ -9,9 +9,9 @@ class [[nodiscard]] BasicScopeGuard
|
||||
{
|
||||
public:
|
||||
constexpr BasicScopeGuard() = default;
|
||||
constexpr BasicScopeGuard(BasicScopeGuard && src) : function{src.release()} {} // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor)
|
||||
constexpr BasicScopeGuard(BasicScopeGuard && src) : function{src.release()} {} // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor, cppcoreguidelines-noexcept-move-operations)
|
||||
|
||||
constexpr BasicScopeGuard & operator=(BasicScopeGuard && src) // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor)
|
||||
constexpr BasicScopeGuard & operator=(BasicScopeGuard && src) // NOLINT(hicpp-noexcept-move, performance-noexcept-move-constructor, cppcoreguidelines-noexcept-move-operations)
|
||||
{
|
||||
if (this != &src)
|
||||
{
|
||||
@ -23,11 +23,11 @@ public:
|
||||
|
||||
template <typename G>
|
||||
requires std::is_convertible_v<G, F>
|
||||
constexpr BasicScopeGuard(BasicScopeGuard<G> && src) : function{src.release()} {} // NOLINT(google-explicit-constructor)
|
||||
constexpr BasicScopeGuard(BasicScopeGuard<G> && src) : function{src.release()} {} // NOLINT(google-explicit-constructor, cppcoreguidelines-rvalue-reference-param-not-moved, cppcoreguidelines-noexcept-move-operations)
|
||||
|
||||
template <typename G>
|
||||
requires std::is_convertible_v<G, F>
|
||||
constexpr BasicScopeGuard & operator=(BasicScopeGuard<G> && src)
|
||||
constexpr BasicScopeGuard & operator=(BasicScopeGuard<G> && src) // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved, cppcoreguidelines-noexcept-move-operations)
|
||||
{
|
||||
if (this != &src)
|
||||
{
|
||||
@ -43,7 +43,7 @@ public:
|
||||
|
||||
template <typename G>
|
||||
requires std::is_convertible_v<G, F>
|
||||
constexpr BasicScopeGuard(G && function_) : function{std::move(function_)} {} // NOLINT(google-explicit-constructor, bugprone-forwarding-reference-overload, bugprone-move-forwarding-reference)
|
||||
constexpr BasicScopeGuard(G && function_) : function{std::move(function_)} {} // NOLINT(google-explicit-constructor, bugprone-forwarding-reference-overload, bugprone-move-forwarding-reference, cppcoreguidelines-missing-std-forward)
|
||||
|
||||
~BasicScopeGuard() { invoke(); }
|
||||
|
||||
@ -70,7 +70,7 @@ public:
|
||||
|
||||
template <typename G>
|
||||
requires std::is_convertible_v<G, F>
|
||||
BasicScopeGuard<F> & join(BasicScopeGuard<G> && other)
|
||||
BasicScopeGuard<F> & join(BasicScopeGuard<G> && other) // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved)
|
||||
{
|
||||
if (other.function)
|
||||
{
|
||||
|
||||
@ -23,10 +23,10 @@ public:
|
||||
constexpr StrongTypedef(): t() {}
|
||||
|
||||
constexpr StrongTypedef(const Self &) = default;
|
||||
constexpr StrongTypedef(Self &&) noexcept(std::is_nothrow_move_constructible_v<T>) = default;
|
||||
constexpr StrongTypedef(Self &&) noexcept(std::is_nothrow_move_constructible_v<T>) = default; // NOLINT(cppcoreguidelines-noexcept-move-operations, hicpp-noexcept-move, performance-noexcept-move-constructor)
|
||||
|
||||
Self & operator=(const Self &) = default;
|
||||
Self & operator=(Self &&) noexcept(std::is_nothrow_move_assignable_v<T>)= default;
|
||||
Self & operator=(Self &&) noexcept(std::is_nothrow_move_assignable_v<T>)= default; // NOLINT(cppcoreguidelines-noexcept-move-operations, hicpp-noexcept-move, performance-noexcept-move-constructor)
|
||||
|
||||
template <class Enable = typename std::is_copy_assignable<T>::type>
|
||||
Self & operator=(const T & rhs) { t = rhs; return *this;}
|
||||
|
||||
@ -463,7 +463,7 @@ auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound)
|
||||
}
|
||||
|
||||
template <typename Iter, typename RandType>
|
||||
void shuffle(Iter from, Iter to, RandType&& rng)
|
||||
void shuffle(Iter from, Iter to, RandType&& rng) // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
{
|
||||
typedef typename std::iterator_traits<Iter>::difference_type delta_t;
|
||||
typedef typename std::remove_reference<RandType>::type::result_type result_t;
|
||||
|
||||
@ -5,14 +5,14 @@ if (ENABLE_CLANG_TIDY)
|
||||
|
||||
find_program (CLANG_TIDY_CACHE_PATH NAMES "clang-tidy-cache")
|
||||
if (CLANG_TIDY_CACHE_PATH)
|
||||
find_program (_CLANG_TIDY_PATH NAMES "clang-tidy-16" "clang-tidy-15" "clang-tidy-14" "clang-tidy")
|
||||
find_program (_CLANG_TIDY_PATH NAMES "clang-tidy-17" "clang-tidy-16" "clang-tidy")
|
||||
|
||||
# Why do we use ';' here?
|
||||
# It's a cmake black magic: https://cmake.org/cmake/help/latest/prop_tgt/LANG_CLANG_TIDY.html#prop_tgt:%3CLANG%3E_CLANG_TIDY
|
||||
# The CLANG_TIDY_PATH is passed to CMAKE_CXX_CLANG_TIDY, which follows CXX_CLANG_TIDY syntax.
|
||||
set (CLANG_TIDY_PATH "${CLANG_TIDY_CACHE_PATH};${_CLANG_TIDY_PATH}" CACHE STRING "A combined command to run clang-tidy with caching wrapper")
|
||||
else ()
|
||||
find_program (CLANG_TIDY_PATH NAMES "clang-tidy-16" "clang-tidy-15" "clang-tidy-14" "clang-tidy")
|
||||
find_program (CLANG_TIDY_PATH NAMES "clang-tidy-17" "clang-tidy-16" "clang-tidy")
|
||||
endif ()
|
||||
|
||||
if (CLANG_TIDY_PATH)
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This is a workaround for bug in llvm/clang,
|
||||
# that does not produce .debug_aranges with LTO
|
||||
#
|
||||
# NOTE: this is a temporary solution, that should be removed after upgrading to
|
||||
# clang-16/llvm-16.
|
||||
#
|
||||
# Refs: https://reviews.llvm.org/D133092
|
||||
|
||||
# NOTE: only -flto=thin is supported.
|
||||
# NOTE: it is not possible to check was there -gdwarf-aranges initially or not.
|
||||
if [[ "$*" =~ -plugin-opt=thinlto ]]; then
|
||||
exec "@LLD_PATH@" -plugin-opt=-generate-arange-section "$@"
|
||||
else
|
||||
exec "@LLD_PATH@" "$@"
|
||||
fi
|
||||
@ -14,15 +14,6 @@ set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER")
|
||||
if (SANITIZE)
|
||||
if (SANITIZE STREQUAL "address")
|
||||
set (ASAN_FLAGS "-fsanitize=address -fsanitize-address-use-after-scope")
|
||||
if (COMPILER_CLANG)
|
||||
if (${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER_EQUAL 15 AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 16)
|
||||
# LLVM-15 has a bug in Address Sanitizer, preventing the usage
|
||||
# of 'sanitize-address-use-after-scope', see [1].
|
||||
#
|
||||
# [1]: https://github.com/llvm/llvm-project/issues/58633
|
||||
set (ASAN_FLAGS "${ASAN_FLAGS} -fno-sanitize-address-use-after-scope")
|
||||
endif()
|
||||
endif()
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}")
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE COMPILER
|
||||
message (STATUS "Using compiler:\n${COMPILER_SELF_IDENTIFICATION}")
|
||||
|
||||
# Require minimum compiler versions
|
||||
set (CLANG_MINIMUM_VERSION 15)
|
||||
set (CLANG_MINIMUM_VERSION 16)
|
||||
set (XCODE_MINIMUM_VERSION 12.0)
|
||||
set (APPLE_CLANG_MINIMUM_VERSION 12.0.0)
|
||||
|
||||
@ -70,16 +70,7 @@ if (LINKER_NAME)
|
||||
if (NOT LLD_PATH)
|
||||
message (FATAL_ERROR "Using linker ${LINKER_NAME} but can't find its path.")
|
||||
endif ()
|
||||
# This a temporary quirk to emit .debug_aranges with ThinLTO, it is only the case clang/llvm <16
|
||||
if (COMPILER_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16)
|
||||
set (LLD_WRAPPER "${CMAKE_CURRENT_BINARY_DIR}/ld.lld")
|
||||
configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ld.lld.in" "${LLD_WRAPPER}" @ONLY)
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}")
|
||||
else ()
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_PATH}")
|
||||
endif()
|
||||
|
||||
endif ()
|
||||
|
||||
if (LINKER_NAME)
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
if (APPLE OR SANITIZE STREQUAL "undefined")
|
||||
if (APPLE OR SANITIZE STREQUAL "undefined" OR SANITIZE STREQUAL "memory")
|
||||
set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF)
|
||||
else()
|
||||
set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON)
|
||||
|
||||
@ -1,9 +1,7 @@
|
||||
{
|
||||
"docker/packager/binary": {
|
||||
"name": "clickhouse/binary-builder",
|
||||
"dependent": [
|
||||
"docker/test/codebrowser"
|
||||
]
|
||||
"dependent": []
|
||||
},
|
||||
"docker/test/compatibility/centos": {
|
||||
"name": "clickhouse/test-old-centos",
|
||||
@ -63,10 +61,6 @@
|
||||
"name": "clickhouse/upgrade-check",
|
||||
"dependent": []
|
||||
},
|
||||
"docker/test/codebrowser": {
|
||||
"name": "clickhouse/codebrowser",
|
||||
"dependent": []
|
||||
},
|
||||
"docker/test/integration/runner": {
|
||||
"only_amd64": true,
|
||||
"name": "clickhouse/integration-tests-runner",
|
||||
|
||||
@ -393,18 +393,18 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--compiler",
|
||||
choices=(
|
||||
"clang-16",
|
||||
"clang-16-darwin",
|
||||
"clang-16-darwin-aarch64",
|
||||
"clang-16-aarch64",
|
||||
"clang-16-aarch64-v80compat",
|
||||
"clang-16-ppc64le",
|
||||
"clang-16-riscv64",
|
||||
"clang-16-s390x",
|
||||
"clang-16-amd64-compat",
|
||||
"clang-16-freebsd",
|
||||
"clang-17",
|
||||
"clang-17-darwin",
|
||||
"clang-17-darwin-aarch64",
|
||||
"clang-17-aarch64",
|
||||
"clang-17-aarch64-v80compat",
|
||||
"clang-17-ppc64le",
|
||||
"clang-17-riscv64",
|
||||
"clang-17-s390x",
|
||||
"clang-17-amd64-compat",
|
||||
"clang-17-freebsd",
|
||||
),
|
||||
default="clang-16",
|
||||
default="clang-17",
|
||||
help="a compiler to use",
|
||||
)
|
||||
parser.add_argument(
|
||||
|
||||
@ -1,30 +0,0 @@
|
||||
# rebuild in #33610
|
||||
# docker build --network=host -t clickhouse/codebrowser .
|
||||
# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser
|
||||
ARG FROM_TAG=latest
|
||||
FROM clickhouse/binary-builder:$FROM_TAG
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update && apt-get --yes --allow-unauthenticated install libclang-${LLVM_VERSION}-dev libmlir-${LLVM_VERSION}-dev
|
||||
|
||||
ARG TARGETARCH
|
||||
RUN arch=${TARGETARCH:-amd64} \
|
||||
&& case $arch in \
|
||||
amd64) rarch=x86_64 ;; \
|
||||
arm64) rarch=aarch64 ;; \
|
||||
*) exit 1 ;; \
|
||||
esac
|
||||
|
||||
# repo versions doesn't work correctly with C++17
|
||||
# also we push reports to s3, so we add index.html to subfolder urls
|
||||
# https://github.com/ClickHouse/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b
|
||||
RUN git clone --branch=master --depth=1 https://github.com/ClickHouse/woboq_codebrowser /woboq_codebrowser \
|
||||
&& cd /woboq_codebrowser \
|
||||
&& cmake . -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=clang-${LLVM_VERSION} -DCLANG_BUILTIN_HEADERS_DIR=/usr/lib/llvm-${LLVM_VERSION}/lib/clang/${LLVM_VERSION}/include \
|
||||
&& ninja
|
||||
|
||||
COPY build.sh /
|
||||
CMD ["bash", "-c", "/build.sh 2>&1"]
|
||||
@ -1,29 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -x -e
|
||||
|
||||
|
||||
STATIC_DATA=${STATIC_DATA:-/woboq_codebrowser/data}
|
||||
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-/build}
|
||||
BUILD_DIRECTORY=${BUILD_DIRECTORY:-/workdir/build}
|
||||
OUTPUT_DIRECTORY=${OUTPUT_DIRECTORY:-/workdir/output}
|
||||
HTML_RESULT_DIRECTORY=${HTML_RESULT_DIRECTORY:-$OUTPUT_DIRECTORY/html_report}
|
||||
SHA=${SHA:-nosha}
|
||||
DATA=${DATA:-https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data}
|
||||
nproc=$(($(nproc) + 2)) # increase parallelism
|
||||
|
||||
read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
|
||||
|
||||
mkdir -p "$BUILD_DIRECTORY" && cd "$BUILD_DIRECTORY"
|
||||
cmake "$SOURCE_DIRECTORY" -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" -DENABLE_WOBOQ_CODEBROWSER=ON "${CMAKE_FLAGS[@]}"
|
||||
mkdir -p "$HTML_RESULT_DIRECTORY"
|
||||
echo 'Filter out too noisy "Error: filename" lines and keep them in full codebrowser_generator.log'
|
||||
/woboq_codebrowser/generator/codebrowser_generator -b "$BUILD_DIRECTORY" -a \
|
||||
-o "$HTML_RESULT_DIRECTORY" --execute-concurrency="$nproc" -p "ClickHouse:$SOURCE_DIRECTORY:$SHA" \
|
||||
-d "$DATA" \
|
||||
|& ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee "$OUTPUT_DIRECTORY/codebrowser_generator.log" \
|
||||
| grep --line-buffered -v ':[0-9]* Error: '
|
||||
cp -r "$STATIC_DATA" "$HTML_RESULT_DIRECTORY/"
|
||||
/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator "$HTML_RESULT_DIRECTORY" \
|
||||
-d "$DATA" |& ts '%Y-%m-%d %H:%M:%S'
|
||||
@ -31,7 +31,11 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \
|
||||
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
|
||||
&& odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
|
||||
&& odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
|
||||
&& rm -rf /tmp/clickhouse-odbc-tmp
|
||||
&& rm -rf /tmp/clickhouse-odbc-tmp \
|
||||
&& mkdir -p /var/lib/clickhouse \
|
||||
&& chmod 777 /var/lib/clickhouse
|
||||
|
||||
# chmod 777 to make the container user independent
|
||||
|
||||
ENV TZ=Europe/Amsterdam
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
@ -9,7 +9,7 @@ trap 'kill $(jobs -pr) ||:' EXIT
|
||||
stage=${stage:-}
|
||||
|
||||
# Compiler version, normally set by Dockerfile
|
||||
export LLVM_VERSION=${LLVM_VERSION:-16}
|
||||
export LLVM_VERSION=${LLVM_VERSION:-17}
|
||||
|
||||
# A variable to pass additional flags to CMake.
|
||||
# Here we explicitly default it to nothing so that bash doesn't complain about
|
||||
@ -28,6 +28,12 @@ FASTTEST_BUILD=$(readlink -f "${FASTTEST_BUILD:-${BUILD:-$FASTTEST_WORKSPACE/bui
|
||||
FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}")
|
||||
FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}")
|
||||
PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH"
|
||||
# Work around for non-existent user
|
||||
if [ "$HOME" == "/" ]; then
|
||||
HOME="$FASTTEST_WORKSPACE/user-home"
|
||||
mkdir -p "$HOME"
|
||||
export HOME
|
||||
fi
|
||||
|
||||
# Export these variables, so that all subsequent invocations of the script
|
||||
# use them, and not try to guess them anew, which leads to weird effects.
|
||||
@ -152,7 +158,11 @@ function clone_submodules
|
||||
)
|
||||
|
||||
git submodule sync
|
||||
git submodule update --jobs=16 --depth 1 --single-branch --init "${SUBMODULES_TO_UPDATE[@]}"
|
||||
git submodule init
|
||||
# --jobs does not work as fast as real parallel running
|
||||
printf '%s\0' "${SUBMODULES_TO_UPDATE[@]}" | \
|
||||
xargs --max-procs=100 --null --no-run-if-empty --max-args=1 \
|
||||
git submodule update --depth 1 --single-branch
|
||||
git submodule foreach git reset --hard
|
||||
git submodule foreach git checkout @ -f
|
||||
git submodule foreach git clean -xfd
|
||||
@ -271,34 +281,12 @@ case "$stage" in
|
||||
;&
|
||||
"clone_root")
|
||||
clone_root
|
||||
|
||||
# Pass control to the script from cloned sources, unless asked otherwise.
|
||||
if ! [ -v FASTTEST_LOCAL_SCRIPT ]
|
||||
then
|
||||
# 'run' stage is deprecated, used for compatibility with old scripts.
|
||||
# Replace with 'clone_submodules' after Nov 1, 2020.
|
||||
# cd and CLICKHOUSE_DIR are also a setup for old scripts, remove as well.
|
||||
# In modern script we undo it by changing back into workspace dir right
|
||||
# away, see below. Remove that as well.
|
||||
cd "$FASTTEST_SOURCE"
|
||||
CLICKHOUSE_DIR=$(pwd)
|
||||
export CLICKHOUSE_DIR
|
||||
stage=run "$FASTTEST_SOURCE/docker/test/fasttest/run.sh"
|
||||
exit $?
|
||||
fi
|
||||
;&
|
||||
"run")
|
||||
# A deprecated stage that is called by old script and equivalent to everything
|
||||
# after cloning root, starting with cloning submodules.
|
||||
;&
|
||||
"clone_submodules")
|
||||
# Recover after being called from the old script that changes into source directory.
|
||||
# See the compatibility hacks in `clone_root` stage above. Remove at the same time,
|
||||
# after Nov 1, 2020.
|
||||
cd "$FASTTEST_WORKSPACE"
|
||||
clone_submodules 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/submodule_log.txt"
|
||||
;&
|
||||
"run_cmake")
|
||||
cd "$FASTTEST_WORKSPACE"
|
||||
run_cmake
|
||||
;&
|
||||
"build")
|
||||
|
||||
@ -17,7 +17,7 @@ stage=${stage:-}
|
||||
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
echo "$script_dir"
|
||||
repo_dir=ch
|
||||
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-16_debug_none_unsplitted_disable_False_binary"}
|
||||
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-17_debug_none_unsplitted_disable_False_binary"}
|
||||
BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
|
||||
|
||||
function git_clone_with_retry
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-16_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
|
||||
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-17_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
|
||||
CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}
|
||||
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-16_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
|
||||
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-17_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"}
|
||||
CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}
|
||||
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@ set -e
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-16_debug_none_unsplitted_disable_False_binary"}
|
||||
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-17_debug_none_unsplitted_disable_False_binary"}
|
||||
BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
|
||||
|
||||
function wget_with_retry
|
||||
|
||||
@ -6,7 +6,7 @@ ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
# 15.0.2
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=16
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=17
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
|
||||
@ -11,14 +11,14 @@ This is intended for continuous integration checks that run on Linux servers. If
|
||||
|
||||
The cross-build for macOS is based on the [Build instructions](../development/build.md), follow them first.
|
||||
|
||||
## Install Clang-16
|
||||
## Install Clang-17
|
||||
|
||||
Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup.
|
||||
For example the commands for Bionic are like:
|
||||
|
||||
``` bash
|
||||
sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-16 main" >> /etc/apt/sources.list
|
||||
sudo apt-get install clang-16
|
||||
sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-17 main" >> /etc/apt/sources.list
|
||||
sudo apt-get install clang-17
|
||||
```
|
||||
|
||||
## Install Cross-Compilation Toolset {#install-cross-compilation-toolset}
|
||||
@ -55,7 +55,7 @@ curl -L 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX1
|
||||
cd ClickHouse
|
||||
mkdir build-darwin
|
||||
cd build-darwin
|
||||
CC=clang-16 CXX=clang++-16 cmake -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar -DCMAKE_INSTALL_NAME_TOOL=${CCTOOLS}/bin/x86_64-apple-darwin-install_name_tool -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake ..
|
||||
CC=clang-17 CXX=clang++-17 cmake -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar -DCMAKE_INSTALL_NAME_TOOL=${CCTOOLS}/bin/x86_64-apple-darwin-install_name_tool -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake ..
|
||||
ninja
|
||||
```
|
||||
|
||||
|
||||
@ -57,7 +57,7 @@ sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
|
||||
|
||||
For other Linux distributions - check the availability of LLVM's [prebuild packages](https://releases.llvm.org/download.html).
|
||||
|
||||
As of April 2023, clang-16 or higher will work.
|
||||
As of August 2023, clang-16 or higher will work.
|
||||
GCC as a compiler is not supported.
|
||||
To build with a specific Clang version:
|
||||
|
||||
@ -67,8 +67,8 @@ to see what version you have installed before setting this environment variable.
|
||||
:::
|
||||
|
||||
``` bash
|
||||
export CC=clang-16
|
||||
export CXX=clang++-16
|
||||
export CC=clang-17
|
||||
export CXX=clang++-17
|
||||
```
|
||||
|
||||
### Checkout ClickHouse Sources {#checkout-clickhouse-sources}
|
||||
@ -105,8 +105,8 @@ The build requires the following components:
|
||||
|
||||
- Git (used to checkout the sources, not needed for the build)
|
||||
- CMake 3.20 or newer
|
||||
- Compiler: clang-16 or newer
|
||||
- Linker: lld-16 or newer
|
||||
- Compiler: clang-17 or newer
|
||||
- Linker: lld-17 or newer
|
||||
- Ninja
|
||||
- Yasm
|
||||
- Gawk
|
||||
|
||||
@ -102,7 +102,7 @@ Builds ClickHouse in various configurations for use in further steps. You have t
|
||||
|
||||
### Report Details
|
||||
|
||||
- **Compiler**: `clang-16`, optionally with the name of a target platform
|
||||
- **Compiler**: `clang-17`, optionally with the name of a target platform
|
||||
- **Build type**: `Debug` or `RelWithDebInfo` (cmake).
|
||||
- **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan).
|
||||
- **Status**: `success` or `fail`
|
||||
|
||||
@ -152,7 +152,7 @@ While inside the `build` directory, configure your build by running CMake. Befor
|
||||
export CC=clang CXX=clang++
|
||||
cmake ..
|
||||
|
||||
If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-16 CXX=clang++-16`. The clang version will be in the script output.
|
||||
If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-17 CXX=clang++-17`. The clang version will be in the script output.
|
||||
|
||||
The `CC` variable specifies the compiler for C (short for C Compiler), and `CXX` variable instructs which C++ compiler is to be used for building.
|
||||
|
||||
@ -276,8 +276,6 @@ Most probably some of the builds will fail at first times. This is due to the fa
|
||||
|
||||
## Browse ClickHouse Source Code {#browse-clickhouse-source-code}
|
||||
|
||||
You can use the **Woboq** online code browser available [here](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). It provides code navigation, semantic highlighting, search and indexing. The code snapshot is updated daily.
|
||||
|
||||
You can use GitHub integrated code browser [here](https://github.dev/ClickHouse/ClickHouse).
|
||||
|
||||
Also, you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual.
|
||||
|
||||
@ -203,9 +203,10 @@ Parameter `NumTrees` is the number of trees which the algorithm creates (default
|
||||
more accurate search results but slower index creation / query times (approximately linearly) as well as larger index sizes.
|
||||
|
||||
:::note
|
||||
Indexes over columns of type `Array` will generally work faster than indexes on `Tuple` columns. All arrays **must** have same length. Use
|
||||
[CONSTRAINT](/docs/en/sql-reference/statements/create/table.md#constraints) to avoid errors. For example, `CONSTRAINT constraint_name_1
|
||||
CHECK length(vectors) = 256`.
|
||||
Indexes over columns of type `Array` will generally work faster than indexes on `Tuple` columns. All arrays must have same length. To avoid
|
||||
errors, you can use a [CONSTRAINT](/docs/en/sql-reference/statements/create/table.md#constraints), for example, `CONSTRAINT
|
||||
constraint_name_1 CHECK length(vectors) = 256`. Also, empty `Arrays` and unspecified `Array` values in INSERT statements (i.e. default
|
||||
values) are not supported.
|
||||
:::
|
||||
|
||||
Setting `annoy_index_search_k_nodes` (default: `NumTrees * LIMIT`) determines how many tree nodes are inspected during SELECTs. Larger
|
||||
@ -223,6 +224,7 @@ SETTINGS annoy_index_search_k_nodes=100;
|
||||
The Annoy index currently does not work with per-table, non-default `index_granularity` settings (see
|
||||
[here](https://github.com/ClickHouse/ClickHouse/pull/51325#issuecomment-1605920475)). If necessary, the value must be changed in config.xml.
|
||||
:::
|
||||
|
||||
## USearch {#usearch}
|
||||
|
||||
This type of ANN index is based on the [the USearch library](https://github.com/unum-cloud/usearch), which implements the [HNSW
|
||||
|
||||
@ -1354,3 +1354,4 @@ In this sample configuration:
|
||||
- `_part_uuid` — Unique part identifier (if enabled MergeTree setting `assign_part_uuids`).
|
||||
- `_partition_value` — Values (a tuple) of a `partition by` expression.
|
||||
- `_sample_factor` — Sample factor (from the query).
|
||||
- `_block_number` — Block number of the row, it is persisted on merges when `allow_experimental_block_number_column` is set to true.
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
slug: /en/interfaces/third-party/client-libraries
|
||||
sidebar_position: 26
|
||||
sidebar_label: Client Libraries
|
||||
description: Third-party client libraries
|
||||
---
|
||||
|
||||
# Client Libraries from Third-party Developers
|
||||
@ -10,13 +11,13 @@ sidebar_label: Client Libraries
|
||||
ClickHouse Inc does **not** maintain the libraries listed below and hasn’t done any extensive testing to ensure their quality.
|
||||
:::
|
||||
|
||||
- Python
|
||||
### Python
|
||||
- [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)
|
||||
- [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
|
||||
- [clickhouse-client](https://github.com/yurial/clickhouse-client)
|
||||
- [aiochclient](https://github.com/maximdanilchenko/aiochclient)
|
||||
- [asynch](https://github.com/long2ice/asynch)
|
||||
- PHP
|
||||
### PHP
|
||||
- [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
|
||||
- [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
|
||||
- [bozerkins/clickhouse-client](https://packagist.org/packages/bozerkins/clickhouse-client)
|
||||
@ -27,51 +28,51 @@ ClickHouse Inc does **not** maintain the libraries listed below and hasn’t don
|
||||
- [glushkovds/phpclickhouse-laravel](https://packagist.org/packages/glushkovds/phpclickhouse-laravel)
|
||||
- [kolya7k ClickHouse PHP extension](https://github.com//kolya7k/clickhouse-php)
|
||||
- [hyvor/clickhouse-php](https://github.com/hyvor/clickhouse-php)
|
||||
- Go
|
||||
### Go
|
||||
- [clickhouse](https://github.com/kshvakov/clickhouse/)
|
||||
- [go-clickhouse](https://github.com/roistat/go-clickhouse)
|
||||
- [chconn](https://github.com/vahid-sohrabloo/chconn)
|
||||
- [mailrugo-clickhouse](https://github.com/mailru/go-clickhouse)
|
||||
- [golang-clickhouse](https://github.com/leprosus/golang-clickhouse)
|
||||
- [uptrace/go-clickhouse](https://clickhouse.uptrace.dev/)
|
||||
- Swift
|
||||
### Swift
|
||||
- [ClickHouseNIO](https://github.com/patrick-zippenfenig/ClickHouseNIO)
|
||||
- [ClickHouseVapor ORM](https://github.com/patrick-zippenfenig/ClickHouseVapor)
|
||||
- NodeJs
|
||||
### NodeJs
|
||||
- [clickhouse (NodeJs)](https://github.com/TimonKK/clickhouse)
|
||||
- [node-clickhouse](https://github.com/apla/node-clickhouse)
|
||||
- [nestjs-clickhouse](https://github.com/depyronick/nestjs-clickhouse)
|
||||
- [clickhouse-client](https://github.com/depyronick/clickhouse-client)
|
||||
- [node-clickhouse-orm](https://github.com/zimv/node-clickhouse-orm)
|
||||
- Perl
|
||||
### Perl
|
||||
- [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse)
|
||||
- [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse)
|
||||
- [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse)
|
||||
- Ruby
|
||||
### Ruby
|
||||
- [ClickHouse (Ruby)](https://github.com/shlima/click_house)
|
||||
- [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
|
||||
- Rust
|
||||
### Rust
|
||||
- [clickhouse.rs](https://github.com/loyd/clickhouse.rs)
|
||||
- [clickhouse-rs](https://github.com/suharev7/clickhouse-rs)
|
||||
- [Klickhouse](https://github.com/Protryon/klickhouse)
|
||||
- R
|
||||
### R
|
||||
- [RClickHouse](https://github.com/IMSMWU/RClickHouse)
|
||||
- Java
|
||||
### Java
|
||||
- [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java)
|
||||
- [clickhouse-client](https://github.com/Ecwid/clickhouse-client)
|
||||
- Scala
|
||||
### Scala
|
||||
- [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client)
|
||||
- Kotlin
|
||||
### Kotlin
|
||||
- [AORM](https://github.com/TanVD/AORM)
|
||||
- C#
|
||||
### C#
|
||||
- [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient)
|
||||
- [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net)
|
||||
- [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client)
|
||||
- [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net)
|
||||
- Elixir
|
||||
### Elixir
|
||||
- [clickhousex](https://github.com/appodeal/clickhousex/)
|
||||
- [pillar](https://github.com/sofakingworld/pillar)
|
||||
- Nim
|
||||
### Nim
|
||||
- [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse)
|
||||
- Haskell
|
||||
### Haskell
|
||||
- [hdbc-clickhouse](https://github.com/zaneli/hdbc-clickhouse)
|
||||
|
||||
@ -142,7 +142,7 @@ As a result, the query cache stores for each query multiple (partial)
|
||||
result blocks. While this behavior is a good default, it can be suppressed using setting
|
||||
[query_cache_squash_partial_results](settings/settings.md#query-cache-squash-partial-results).
|
||||
|
||||
Also, results of queries with non-deterministic functions are not cached. Such functions include
|
||||
Also, results of queries with non-deterministic functions are not cached by default. Such functions include
|
||||
- functions for accessing dictionaries: [`dictGet()`](../sql-reference/functions/ext-dict-functions.md#dictGet) etc.
|
||||
- [user-defined functions](../sql-reference/statements/create/function.md),
|
||||
- functions which return the current date or time: [`now()`](../sql-reference/functions/date-time-functions.md#now),
|
||||
@ -158,7 +158,7 @@ Also, results of queries with non-deterministic functions are not cached. Such f
|
||||
- functions which depend on the environment: [`currentUser()`](../sql-reference/functions/other-functions.md#currentUser),
|
||||
[`queryID()`](../sql-reference/functions/other-functions.md#queryID),
|
||||
[`getMacro()`](../sql-reference/functions/other-functions.md#getMacro) etc.
|
||||
Caching of non-deterministic functions can be forced regardless using setting
|
||||
To force caching of results of queries with non-deterministic functions regardless, use setting
|
||||
[query_cache_store_results_of_queries_with_nondeterministic_functions](settings/settings.md#query-cache-store-results-of-queries-with-nondeterministic-functions).
|
||||
|
||||
Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a
|
||||
|
||||
@ -854,3 +854,9 @@ Possible values:
|
||||
- `Always` or `Never`.
|
||||
|
||||
Default value: `Never`
|
||||
|
||||
## allow_experimental_block_number_column
|
||||
|
||||
Persists virtual column `_block_number` on merges.
|
||||
|
||||
Default value: false.
|
||||
|
||||
@ -4067,6 +4067,17 @@ Result:
|
||||
└─────┴─────┴───────┘
|
||||
```
|
||||
|
||||
## splitby_max_substrings_includes_remaining_string {#splitby_max_substrings_includes_remaining_string}
|
||||
|
||||
Controls whether function [splitBy*()](../../sql-reference/functions/splitting-merging-functions.md) with argument `max_substrings` > 0 will include the remaining string in the last element of the result array.
|
||||
|
||||
Possible values:
|
||||
|
||||
- `0` - The remaining string will not be included in the last element of the result array.
|
||||
- `1` - The remaining string will be included in the last element of the result array. This is the behavior of Spark's [`split()`](https://spark.apache.org/docs/3.1.2/api/python/reference/api/pyspark.sql.functions.split.html) function and Python's ['string.split()'](https://docs.python.org/3/library/stdtypes.html#str.split) method.
|
||||
|
||||
Default value: `0`
|
||||
|
||||
## enable_extended_results_for_datetime_functions {#enable-extended-results-for-datetime-functions}
|
||||
|
||||
Enables or disables returning results of type:
|
||||
@ -4667,44 +4678,3 @@ The default value is `false`.
|
||||
``` xml
|
||||
<validate_tcp_client_information>true</validate_tcp_client_information>
|
||||
```
|
||||
|
||||
## ignore_access_denied_multidirectory_globs {#ignore_access_denied_multidirectory_globs}
|
||||
|
||||
Allows to ignore 'permission denied' errors when using multi-directory `{}` globs for [File](../../sql-reference/table-functions/file.md#globs_in_path) and [HDFS](../../sql-reference/table-functions/hdfs.md) storages.
|
||||
This setting is only applicable to multi directory `{}` globs.
|
||||
|
||||
Possible values: `0`, `1`.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
### Example
|
||||
|
||||
Having the following structure in `user_files`:
|
||||
```
|
||||
my_directory/
|
||||
├── data1
|
||||
│ ├── f1.csv
|
||||
├── data2
|
||||
│ ├── f2.csv
|
||||
└── test_root
|
||||
```
|
||||
where `data1`, `data2` directories are accessible, but one has no rights to read `test_root` directories.
|
||||
|
||||
For a query like `SELECT *, _path, _file FROM file('my_directory/{data1/f1,data2/f2}.csv', CSV)` an exception will be thrown:
|
||||
`Code: 1001. DB::Exception: std::__1::__fs::filesystem::filesystem_error: filesystem error: in directory_iterator::directory_iterator(...): Permission denied`.
|
||||
It happens because a multi-directory glob requires a recursive search in _all_ available directories under `my_directory`.
|
||||
|
||||
If this setting is on, all inaccessible directories will be silently skipped, even if they are explicitly specified inside `{}`.
|
||||
|
||||
```sql
|
||||
SELECT _path, _file FROM file('my_directory/{data1/f1,data2/f2}.csv', CSV) SETTINGS ignore_access_denied_multidirectory_globs = 0;
|
||||
|
||||
Code: 1001. DB::Exception: std::__1::__fs::filesystem::filesystem_error: filesystem error: in directory_iterator::directory_iterator(...): Permission denied
|
||||
```
|
||||
```sql
|
||||
SELECT _path, _file FROM file('my_directory/{data1/f1,data2/f2}.csv', CSV) SETTINGS ignore_access_denied_multidirectory_globs = 1;
|
||||
|
||||
┌─_path───────────────────┬─_file───────┐
|
||||
│ <full path to file> │ <file name> │
|
||||
└─────────────────────────┴─────────────┘
|
||||
```
|
||||
|
||||
@ -13,16 +13,20 @@ SHOW TABLES FROM information_schema;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─name─────┐
|
||||
┌─name────────────────────┐
|
||||
│ COLUMNS │
|
||||
│ KEY_COLUMN_USAGE │
|
||||
│ REFERENTIAL_CONSTRAINTS │
|
||||
│ SCHEMATA │
|
||||
│ TABLES │
|
||||
│ VIEWS │
|
||||
│ columns │
|
||||
│ key_column_usage │
|
||||
│ referential_constraints │
|
||||
│ schemata │
|
||||
│ tables │
|
||||
│ views │
|
||||
└──────────┘
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
`INFORMATION_SCHEMA` contains the following views:
|
||||
@ -31,8 +35,10 @@ SHOW TABLES FROM information_schema;
|
||||
- [SCHEMATA](#schemata)
|
||||
- [TABLES](#tables)
|
||||
- [VIEWS](#views)
|
||||
- [KEY_COLUMN_USAGE](#key_column_usage)
|
||||
- [REFERENTIAL_CONSTRAINTS](#referential_constraints)
|
||||
|
||||
Case-insensitive equivalent views, e.g. `INFORMATION_SCHEMA.columns` are provided for reasons of compatibility with other databases.
|
||||
Case-insensitive equivalent views, e.g. `INFORMATION_SCHEMA.columns` are provided for reasons of compatibility with other databases. The same applies to all the columns in these views - both lowercase (for example, `table_name`) and uppercase (`TABLE_NAME`) variants are provided.
|
||||
|
||||
## COLUMNS {#columns}
|
||||
|
||||
@ -69,7 +75,36 @@ Columns:
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE (table_schema=currentDatabase() OR table_schema='') AND table_name NOT LIKE '%inner%' LIMIT 1 FORMAT Vertical;
|
||||
SELECT table_catalog,
|
||||
table_schema,
|
||||
table_name,
|
||||
column_name,
|
||||
ordinal_position,
|
||||
column_default,
|
||||
is_nullable,
|
||||
data_type,
|
||||
character_maximum_length,
|
||||
character_octet_length,
|
||||
numeric_precision,
|
||||
numeric_precision_radix,
|
||||
numeric_scale,
|
||||
datetime_precision,
|
||||
character_set_catalog,
|
||||
character_set_schema,
|
||||
character_set_name,
|
||||
collation_catalog,
|
||||
collation_schema,
|
||||
collation_name,
|
||||
domain_catalog,
|
||||
domain_schema,
|
||||
domain_name,
|
||||
column_comment,
|
||||
column_type
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE (table_schema = currentDatabase() OR table_schema = '')
|
||||
AND table_name NOT LIKE '%inner%'
|
||||
LIMIT 1
|
||||
FORMAT Vertical;
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -121,7 +156,17 @@ Columns:
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM information_schema.schemata WHERE schema_name ILIKE 'information_schema' LIMIT 1 FORMAT Vertical;
|
||||
SELECT catalog_name,
|
||||
schema_name,
|
||||
schema_owner,
|
||||
default_character_set_catalog,
|
||||
default_character_set_schema,
|
||||
default_character_set_name,
|
||||
sql_path
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name ilike 'information_schema'
|
||||
LIMIT 1
|
||||
FORMAT Vertical;
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -147,19 +192,31 @@ Columns:
|
||||
- `table_catalog` ([String](../../sql-reference/data-types/string.md)) — The name of the database in which the table is located.
|
||||
- `table_schema` ([String](../../sql-reference/data-types/string.md)) — The name of the database in which the table is located.
|
||||
- `table_name` ([String](../../sql-reference/data-types/string.md)) — Table name.
|
||||
- `table_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Table type. Possible values:
|
||||
- `table_type` ([String](../../sql-reference/data-types/string.md)) — Table type. Possible values:
|
||||
- `BASE TABLE`
|
||||
- `VIEW`
|
||||
- `FOREIGN TABLE`
|
||||
- `LOCAL TEMPORARY`
|
||||
- `SYSTEM VIEW`
|
||||
- `table_comment` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — The comment used when creating the table.
|
||||
- `table_collation` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — The table default collation. Always `utf8mb4`.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE (table_schema = currentDatabase() OR table_schema = '') AND table_name NOT LIKE '%inner%' LIMIT 1 FORMAT Vertical;
|
||||
SELECT table_catalog,
|
||||
table_schema,
|
||||
table_name,
|
||||
table_type,
|
||||
table_collation,
|
||||
table_comment
|
||||
FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE (table_schema = currentDatabase() OR table_schema = '')
|
||||
AND table_name NOT LIKE '%inner%'
|
||||
LIMIT 1
|
||||
FORMAT Vertical;
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -171,6 +228,8 @@ table_catalog: default
|
||||
table_schema: default
|
||||
table_name: describe_example
|
||||
table_type: BASE TABLE
|
||||
table_collation: utf8mb4_0900_ai_ci
|
||||
table_comment:
|
||||
```
|
||||
|
||||
## VIEWS {#views}
|
||||
@ -199,7 +258,20 @@ Query:
|
||||
``` sql
|
||||
CREATE VIEW v (n Nullable(Int32), f Float64) AS SELECT n, f FROM t;
|
||||
CREATE MATERIALIZED VIEW mv ENGINE = Null AS SELECT * FROM system.one;
|
||||
SELECT * FROM information_schema.views WHERE table_schema = currentDatabase() LIMIT 1 FORMAT Vertical;
|
||||
SELECT table_catalog,
|
||||
table_schema,
|
||||
table_name,
|
||||
view_definition,
|
||||
check_option,
|
||||
is_updatable,
|
||||
is_insertable_into,
|
||||
is_trigger_updatable,
|
||||
is_trigger_deletable,
|
||||
is_trigger_insertable_into
|
||||
FROM information_schema.views
|
||||
WHERE table_schema = currentDatabase()
|
||||
LIMIT 1
|
||||
FORMAT Vertical;
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -218,3 +290,80 @@ is_trigger_updatable: NO
|
||||
is_trigger_deletable: NO
|
||||
is_trigger_insertable_into: NO
|
||||
```
|
||||
|
||||
## KEY_COLUMN_USAGE (#key_column_usage)
|
||||
|
||||
Contains columns from the [system.tables](../../operations/system-tables/tables.md) system table which are restricted by constraints.
|
||||
|
||||
Columns:
|
||||
|
||||
- `constraint_catalog` ([String](../../sql-reference/data-types/string.md)) — Currently unused. Always `def`.
|
||||
- `constraint_schema` ([String](../../sql-reference/data-types/string.md)) — The name of the schema (database) to which the constraint belongs.
|
||||
- `constraint_name` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — The name of the constraint.
|
||||
- `table_catalog` ([String](../../sql-reference/data-types/string.md)) — Currently unused. Always `def`.
|
||||
- `table_schema` ([String](../../sql-reference/data-types/string.md)) — The name of the schema (database) to which the table belongs.
|
||||
- `table_name` ([String](../../sql-reference/data-types/string.md)) — The name of the table that has the constraint.
|
||||
- `column_name` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — The name of the column that has the constraint.
|
||||
- `ordinal_position` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Currently unused. Always `1`.
|
||||
- `position_in_unique_constraint` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt32](../../sql-reference/data-types/int-uint.md))) — Currently unused. Always `NULL`.
|
||||
- `referenced_table_schema` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Currently unused. Always NULL.
|
||||
- `referenced_table_name` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Currently unused. Always NULL.
|
||||
- `referenced_column_name` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Currently unused. Always NULL.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
CREATE TABLE test (i UInt32, s String) ENGINE MergeTree ORDER BY i;
|
||||
SELECT constraint_catalog,
|
||||
constraint_schema,
|
||||
constraint_name,
|
||||
table_catalog,
|
||||
table_schema,
|
||||
table_name,
|
||||
column_name,
|
||||
ordinal_position,
|
||||
position_in_unique_constraint,
|
||||
referenced_table_schema,
|
||||
referenced_table_name,
|
||||
referenced_column_name
|
||||
FROM information_schema.key_column_usage
|
||||
WHERE table_name = 'test'
|
||||
FORMAT Vertical;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
Row 1:
|
||||
──────
|
||||
constraint_catalog: def
|
||||
constraint_schema: default
|
||||
constraint_name: PRIMARY
|
||||
table_catalog: def
|
||||
table_schema: default
|
||||
table_name: test
|
||||
column_name: i
|
||||
ordinal_position: 1
|
||||
position_in_unique_constraint: ᴺᵁᴸᴸ
|
||||
referenced_table_schema: ᴺᵁᴸᴸ
|
||||
referenced_table_name: ᴺᵁᴸᴸ
|
||||
referenced_column_name: ᴺᵁᴸᴸ
|
||||
```
|
||||
|
||||
## REFERENTIAL_CONSTRAINTS (#referential_constraints)
|
||||
|
||||
Contains information about foreign keys. Currently returns an empty result (no rows) which is just enough to provide compatibility with 3rd party tools like Tableau Online.
|
||||
|
||||
Columns:
|
||||
|
||||
- `constraint_catalog` ([String](../../sql-reference/data-types/string.md)) — Currently unused.
|
||||
- `constraint_schema` ([String](../../sql-reference/data-types/string.md)) — Currently unused.
|
||||
- `constraint_name` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Currently unused.
|
||||
- `unique_constraint_catalog` ([String](../../sql-reference/data-types/string.md)) — Currently unused.
|
||||
- `unique_constraint_schema` ([String](../../sql-reference/data-types/string.md)) — Currently unused.
|
||||
- `unique_constraint_name` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Currently unused.
|
||||
- `match_option` ([String](../../sql-reference/data-types/string.md)) — Currently unused.
|
||||
- `update_rule` ([String](../../sql-reference/data-types/string.md)) — Currently unused.
|
||||
- `delete_rule` ([String](../../sql-reference/data-types/string.md)) — Currently unused.
|
||||
- `table_name` ([String](../../sql-reference/data-types/string.md)) — Currently unused.
|
||||
- `referenced_table_name` ([String](../../sql-reference/data-types/string.md)) — Currently unused.
|
||||
|
||||
@ -38,6 +38,10 @@ inflight_requests: ᴺᵁᴸᴸ
|
||||
inflight_cost: ᴺᵁᴸᴸ
|
||||
max_requests: ᴺᵁᴸᴸ
|
||||
max_cost: ᴺᵁᴸᴸ
|
||||
max_speed: ᴺᵁᴸᴸ
|
||||
max_burst: ᴺᵁᴸᴸ
|
||||
throttling_us: ᴺᵁᴸᴸ
|
||||
tokens: ᴺᵁᴸᴸ
|
||||
```
|
||||
|
||||
Columns:
|
||||
@ -62,3 +66,7 @@ Columns:
|
||||
- `inflight_cost` (`Nullable(Int64)`) - For `inflight_limit` nodes only. The sum of costs (e.g. bytes) of all resource requests dequeued from this node, that are currently in consumption state.
|
||||
- `max_requests` (`Nullable(Int64)`) - For `inflight_limit` nodes only. Upper limit for `inflight_requests` leading to constraint violation.
|
||||
- `max_cost` (`Nullable(Int64)`) - For `inflight_limit` nodes only. Upper limit for `inflight_cost` leading to constraint violation.
|
||||
- `max_speed` (`Nullable(Float64)`) - For `bandwidth_limit` nodes only. Upper limit for bandwidth in tokens per second.
|
||||
- `max_burst` (`Nullable(Float64)`) - For `bandwidth_limit` nodes only. Upper limit for `tokens` available in token-bucket throttler.
|
||||
- `throttling_us` (`Nullable(Int64)`) - For `bandwidth_limit` nodes only. Total number of microseconds this node was in throttling state.
|
||||
- `tokens` (`Nullable(Float64)`) - For `bandwidth_limit` nodes only. Number of tokens currently available in token-bucket throttler.
|
||||
|
||||
@ -33,7 +33,7 @@ Columns:
|
||||
- `MemoryPeak` represents collecting updates of peak memory usage.
|
||||
- `ProfileEvent` represents collecting of increments of profile events.
|
||||
|
||||
- `thread_number` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Thread identifier.
|
||||
- `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Thread identifier.
|
||||
|
||||
- `query_id` ([String](../../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query_log](#system_tables-query_log) system table.
|
||||
|
||||
|
||||
@ -79,10 +79,13 @@ graph TD
|
||||
|
||||
**Possible node types:**
|
||||
* `inflight_limit` (constraint) - blocks if either number of concurrent in-flight requests exceeds `max_requests`, or their total cost exceeds `max_cost`; must have a single child.
|
||||
* `bandwidth_limit` (constraint) - blocks if current bandwidth exceeds `max_speed` (0 means unlimited) or burst exceeds `max_burst` (by default equals `max_speed`); must have a single child.
|
||||
* `fair` (policy) - selects the next request to serve from one of its children nodes according to max-min fairness; children nodes can specify `weight` (default is 1).
|
||||
* `priority` (policy) - selects the next request to serve from one of its children nodes according to static priorities (lower value means higher priority); children nodes can specify `priority` (default is 0).
|
||||
* `fifo` (queue) - leaf of the hierarchy capable of holding requests that exceed resource capacity.
|
||||
|
||||
To be able to use the full capacity of the underlying resource, you should use `inflight_limit`. Note that a low number of `max_requests` or `max_cost` could lead to not full resource utilization, while too high numbers could lead to empty queues inside the scheduler, which in turn will result in policies being ignored (unfairness or ignoring of priorities) in the subtree. On the other hand, if you want to protect resources from too high utilization, you should use `bandwidth_limit`. It throttles when the amount of resource consumed in `duration` seconds exceeds `max_burst + max_speed * duration` bytes. Two `bandwidth_limit` nodes on the same resource could be used to limit peak bandwidth during short intervals and average bandwidth for longer ones.
|
||||
|
||||
The following example shows how to define IO scheduling hierarchies shown in the picture:
|
||||
|
||||
```xml
|
||||
|
||||
@ -4,7 +4,7 @@ sidebar_position: 52
|
||||
sidebar_label: Array(T)
|
||||
---
|
||||
|
||||
# Array(t)
|
||||
# Array(T)
|
||||
|
||||
An array of `T`-type items, with the starting array index as 1. `T` can be any data type, including an array.
|
||||
|
||||
|
||||
@ -732,14 +732,15 @@ Returns for a given date, the number of days passed since [1 January 0000](https
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
toDaysSinceYearZero(date)
|
||||
toDaysSinceYearZero(date[, time_zone])
|
||||
```
|
||||
|
||||
Aliases: `TO_DAYS`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `date` — The date to calculate the number of days passed since year zero from. [Date](../../sql-reference/data-types/date.md) or [Date32](../../sql-reference/data-types/date32.md).
|
||||
**Arguments**
|
||||
- `date` — The date to calculate the number of days passed since year zero from. [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md).
|
||||
- `time_zone` — A String type const value or a expression represent the time zone. [String types](../../sql-reference/data-types/string.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
|
||||
@ -660,21 +660,26 @@ SELECT
|
||||
|
||||
## formatReadableTimeDelta
|
||||
|
||||
Given a time interval (delta) in seconds, this function returns a time delta with year/month/day/hour/minute/second as string.
|
||||
Given a time interval (delta) in seconds, this function returns a time delta with year/month/day/hour/minute/second/millisecond/microsecond/nanosecond as string.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
formatReadableTimeDelta(column[, maximum_unit])
|
||||
formatReadableTimeDelta(column[, maximum_unit, minimum_unit])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `column` — A column with a numeric time delta.
|
||||
- `maximum_unit` — Optional. Maximum unit to show. Acceptable values `seconds`, `minutes`, `hours`, `days`, `months`, `years`.
|
||||
|
||||
Example:
|
||||
- `maximum_unit` — Optional. Maximum unit to show.
|
||||
* Acceptable values: `nanoseconds`, `microseconds`, `milliseconds`, `seconds`, `minutes`, `hours`, `days`, `months`, `years`.
|
||||
* Default value: `years`.
|
||||
- `minimum_unit` — Optional. Minimum unit to show. All smaller units are truncated.
|
||||
* Acceptable values: `nanoseconds`, `microseconds`, `milliseconds`, `seconds`, `minutes`, `hours`, `days`, `months`, `years`.
|
||||
* If explicitly specified value is bigger than `maximum_unit`, an exception will be thrown.
|
||||
* Default value: `seconds` if `maximum_unit` is `seconds` or bigger, `nanoseconds` otherwise.
|
||||
|
||||
**Example**
|
||||
``` sql
|
||||
SELECT
|
||||
arrayJoin([100, 12345, 432546534]) AS elapsed,
|
||||
@ -703,6 +708,20 @@ SELECT
|
||||
└────────────┴─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
arrayJoin([100, 12345, 432546534.00000006]) AS elapsed,
|
||||
formatReadableTimeDelta(elapsed, 'minutes', 'nanoseconds') AS time_delta
|
||||
```
|
||||
|
||||
```text
|
||||
┌────────────elapsed─┬─time_delta─────────────────────────────────────┐
|
||||
│ 100 │ 1 minute and 40 seconds │
|
||||
│ 12345 │ 205 minutes and 45 seconds │
|
||||
│ 432546534.00000006 │ 7209108 minutes, 54 seconds and 60 nanoseconds │
|
||||
└────────────────────┴────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## parseTimeDelta
|
||||
|
||||
Parse a sequence of numbers followed by something resembling a time unit.
|
||||
|
||||
@ -21,7 +21,7 @@ splitByChar(separator, s[, max_substrings]))
|
||||
|
||||
- `separator` — The separator which should contain exactly one character. [String](../../sql-reference/data-types/string.md).
|
||||
- `s` — The string to split. [String](../../sql-reference/data-types/string.md).
|
||||
- `max_substrings` — An optional `Int64` defaulting to 0. When `max_substrings` > 0, the returned substrings will be no more than `max_substrings`, otherwise the function will return as many substrings as possible.
|
||||
- `max_substrings` — An optional `Int64` defaulting to 0. If `max_substrings` > 0, the returned array will contain at most `max_substrings` substrings, otherwise the function will return as many substrings as possible.
|
||||
|
||||
**Returned value(s)**
|
||||
|
||||
@ -38,6 +38,10 @@ The behavior of parameter `max_substrings` changed starting with ClickHouse v22.
|
||||
For example,
|
||||
- in v22.10: `SELECT splitByChar('=', 'a=b=c=d', 2); -- ['a','b','c=d']`
|
||||
- in v22.11: `SELECT splitByChar('=', 'a=b=c=d', 2); -- ['a','b']`
|
||||
|
||||
A behavior similar to ClickHouse pre-v22.11 can be achieved by setting
|
||||
[splitby_max_substrings_includes_remaining_string](../../operations/settings/settings.md#splitby_max_substrings_includes_remaining_string)
|
||||
`SELECT splitByChar('=', 'a=b=c=d', 2) SETTINGS splitby_max_substrings_includes_remaining_string = 1 -- ['a', 'b=c=d']`
|
||||
:::
|
||||
|
||||
**Example**
|
||||
@ -80,6 +84,8 @@ Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-refere
|
||||
- There are multiple consecutive non-empty separators;
|
||||
- The original string `s` is empty while the separator is not empty.
|
||||
|
||||
Setting [splitby_max_substrings_includes_remaining_string](../../operations/settings/settings.md#splitby_max_substrings_includes_remaining_string) (default: 0) controls if the remaining string is included in the last element of the result array when argument `max_substrings` > 0.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
@ -133,6 +139,8 @@ Returns an array of selected substrings. Empty substrings may be selected when:
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
Setting [splitby_max_substrings_includes_remaining_string](../../operations/settings/settings.md#splitby_max_substrings_includes_remaining_string) (default: 0) controls if the remaining string is included in the last element of the result array when argument `max_substrings` > 0.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
@ -182,6 +190,8 @@ Returns an array of selected substrings.
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
Setting [splitby_max_substrings_includes_remaining_string](../../operations/settings/settings.md#splitby_max_substrings_includes_remaining_string) (default: 0) controls if the remaining string is included in the last element of the result array when argument `max_substrings` > 0.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
@ -219,6 +229,8 @@ Returns an array of selected substrings.
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
Setting [splitby_max_substrings_includes_remaining_string](../../operations/settings/settings.md#splitby_max_substrings_includes_remaining_string) (default: 0) controls if the remaining string is included in the last element of the result array when argument `max_substrings` > 0.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
@ -279,6 +291,8 @@ Returns an array of selected substrings.
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
Setting [splitby_max_substrings_includes_remaining_string](../../operations/settings/settings.md#splitby_max_substrings_includes_remaining_string) (default: 0) controls if the remaining string is included in the last element of the result array when argument `max_substrings` > 0.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
|
||||
@ -14,6 +14,7 @@ ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
|
||||
[NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'}]
|
||||
[[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[VALID UNTIL datetime]
|
||||
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
||||
[GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
|
||||
@ -14,6 +14,7 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] ...]
|
||||
[NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'}]
|
||||
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[VALID UNTIL datetime]
|
||||
[IN access_storage_type]
|
||||
[DEFAULT ROLE role [,...]]
|
||||
[DEFAULT DATABASE database | NONE]
|
||||
@ -162,6 +163,16 @@ Another way of specifying host is to use `@` syntax following the username. Exam
|
||||
ClickHouse treats `user_name@'address'` as a username as a whole. Thus, technically you can create multiple users with the same `user_name` and different constructions after `@`. However, we do not recommend to do so.
|
||||
:::
|
||||
|
||||
## VALID UNTIL Clause
|
||||
|
||||
Allows you to specify the expiration date and, optionally, the time for user credentials. It accepts a string as a parameter. It is recommended to use the `YYYY-MM-DD [hh:mm:ss] [timezone]` format for datetime. By default, this parameter equals `'infinity'`.
|
||||
|
||||
Examples:
|
||||
|
||||
- `CREATE USER name1 VALID UNTIL '2025-01-01'`
|
||||
- `CREATE USER name1 VALID UNTIL '2025-01-01 12:00:00 UTC'`
|
||||
- `CREATE USER name1 VALID UNTIL 'infinity'`
|
||||
|
||||
## GRANTEES Clause
|
||||
|
||||
Specifies users or roles which are allowed to receive [privileges](../../../sql-reference/statements/grant.md#grant-privileges) from this user on the condition this user has also all required access granted with [GRANT OPTION](../../../sql-reference/statements/grant.md#grant-privigele-syntax). Options of the `GRANTEES` clause:
|
||||
|
||||
@ -21,6 +21,7 @@ DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] [SYNC]
|
||||
## DROP TABLE
|
||||
|
||||
Deletes the table.
|
||||
In case when `IF EMPTY` clause is specified server will check if table is empty only on replica that received initial query.
|
||||
|
||||
:::tip
|
||||
Also see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md)
|
||||
@ -29,7 +30,7 @@ Also see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md)
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] [SYNC]
|
||||
DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY] [db.]name [ON CLUSTER cluster] [SYNC]
|
||||
```
|
||||
|
||||
## DROP DICTIONARY
|
||||
|
||||
@ -141,7 +141,7 @@ Multiple path components can have globs. For being processed file must exist and
|
||||
|
||||
- `*` — Substitutes any number of any characters except `/` including empty string.
|
||||
- `?` — Substitutes any single character.
|
||||
- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. In case at least one of strings contains `/`, `'permission denied'` errors may be ignored using [ignore_access_denied_multidirectory_globs](/docs/en/operations/settings/settings.md#ignore_access_denied_multidirectory_globs) setting for file & HDFS.
|
||||
- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. The strings can contain the `/` symbol.
|
||||
- `{N..M}` — Substitutes any number in range from N to M including both borders.
|
||||
- `**` - Fetches all files inside the folder recursively.
|
||||
|
||||
@ -210,7 +210,6 @@ SELECT count(*) FROM file('big_dir/**/file002', 'CSV', 'name String, value UInt3
|
||||
- [engine_file_allow_create_multiple_files](/docs/en/operations/settings/settings.md#engine_file_allow_create_multiple_files) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [engine_file_skip_empty_files](/docs/en/operations/settings/settings.md#engine_file_skip_empty_files) - allows to skip empty files while reading. Disabled by default.
|
||||
- [storage_file_read_method](/docs/en/operations/settings/settings.md#engine-file-emptyif-not-exists) - method of reading data from storage file, one of: read, pread, mmap (only for clickhouse-local). Default value: `pread` for clickhouse-server, `mmap` for clickhouse-local.
|
||||
- [ignore_access_denied_multidirectory_globs](/docs/en/operations/settings/settings.md#ignore_access_denied_multidirectory_globs) - allows to ignore permission denied errors for multi-directory globs.
|
||||
|
||||
|
||||
**See Also**
|
||||
|
||||
@ -45,10 +45,10 @@ Multiple path components can have globs. For being processed file should exists
|
||||
|
||||
- `*` — Substitutes any number of any characters except `/` including empty string.
|
||||
- `?` — Substitutes any single character.
|
||||
- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. In case at least one of strings contains `/`, `'permission denied'` errors may be ignored using [ignore_access_denied_multidirectory_globs](/docs/en/operations/settings/settings.md#ignore_access_denied_multidirectory_globs) setting.
|
||||
- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. The strings can contain the `/` symbol.
|
||||
- `{N..M}` — Substitutes any number in range from N to M including both borders.
|
||||
|
||||
Constructions with `{}` are similar to the [remote table function](../../sql-reference/table-functions/remote.md)).
|
||||
Constructions with `{}` are similar to the [remote](../../sql-reference/table-functions/remote.md)) table function.
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
@ -289,6 +289,4 @@ Pull request можно создать, даже если работа над з
|
||||
|
||||
## Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse}
|
||||
|
||||
Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно.
|
||||
|
||||
Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse).
|
||||
|
||||
@ -4209,45 +4209,3 @@ SELECT toFloat64('1.7091'), toFloat64('1.5008753E7') SETTINGS precise_float_pars
|
||||
│ 1.7091 │ 15008753 │
|
||||
└─────────────────────┴──────────────────────────┘
|
||||
```
|
||||
|
||||
## ignore_access_denied_multidirectory_globs {#ignore_access_denied_multidirectory_globs}
|
||||
|
||||
Позволяет игнорировать ошибку 'permission denied', возникающую при использовании шаблона `{}`, содержащего `/` внутри себя.
|
||||
Работает для [File](../../sql-reference/table-functions/file.md#globs_in_path) и [HDFS](../../sql-reference/table-functions/hdfs.md).
|
||||
Работает _только_ для указанных выше шаблонов `{}`.
|
||||
|
||||
Возможные значения: `0`, `1`.
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
### Пример
|
||||
|
||||
Пусть в `user_files` имеется следующая структура:
|
||||
```
|
||||
my_directory/
|
||||
├── data1
|
||||
│ ├── f1.csv
|
||||
├── data2
|
||||
│ ├── f2.csv
|
||||
└── test_root
|
||||
```
|
||||
Пусть также директории `data1`, `data2` могут быть прочитаны, но прав на чтение `test_root` нет.
|
||||
|
||||
На запрос `SELECT *, _path, _file FROM file('my_directory/{data1/f1,data2/f2}.csv', CSV)` будет выброшено исключение:
|
||||
`Code: 1001. DB::Exception: std::__1::__fs::filesystem::filesystem_error: filesystem error: in directory_iterator::directory_iterator(...): Permission denied`.
|
||||
Это происходит, так как для обработки такого шаблона необходимо выполнить рекурсивный поиск по _всем_ директориям, находящимся внутри `my_directory`.
|
||||
|
||||
Если данная настройка имеет значение 1, то недоступные директории будут тихо пропущены, даже если они явно указаны внутри `{}`.
|
||||
|
||||
```sql
|
||||
SELECT _path, _file FROM file('my_directory/{data1/f1,data2/f2}.csv', CSV) SETTINGS ignore_access_denied_multidirectory_globs = 0;
|
||||
|
||||
Code: 1001. DB::Exception: std::__1::__fs::filesystem::filesystem_error: filesystem error: in directory_iterator::directory_iterator(...): Permission denied
|
||||
```
|
||||
```sql
|
||||
SELECT _path, _file FROM file('my_directory/{data1/f1,data2/f2}.csv', CSV) SETTINGS ignore_access_denied_multidirectory_globs = 1;
|
||||
|
||||
┌─_path───────────────────┬─_file───────┐
|
||||
│ <full path to file> │ <file name> │
|
||||
└─────────────────────────┴─────────────┘
|
||||
```
|
||||
|
||||
@ -659,9 +659,11 @@ SELECT
|
||||
└────────────────┴───────────────────┘
|
||||
```
|
||||
|
||||
|
||||
## formatReadableTimeDelta {#formatreadabletimedelta}
|
||||
|
||||
Принимает дельту времени в секундах. Возвращает дельту времени с (год, месяц, день, час, минута, секунда) в виде строки.
|
||||
Принимает дельту времени в секундах. Возвращает дельту времени в виде строки.
|
||||
Есть возможность указать наиболее крупную единицу измерения для вывода (год, месяц, день, час, минута, секунда, милли/микро/наносекунда).
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
@ -672,9 +674,15 @@ formatReadableTimeDelta(column[, maximum_unit])
|
||||
**Аргументы**
|
||||
|
||||
- `column` — Столбец с числовой дельтой времени.
|
||||
- `maximum_unit` — Опциональный параметр. Максимальная единица измерения для отображения. Допустимые значения: секунды, минуты, часы, дни, месяцы, годы.
|
||||
- `maximum_unit` — Опциональный параметр. Максимальная единица измерения для отображения.
|
||||
* Допустимые значения: `nanoseconds`, `microseconds`, `milliseconds`, `seconds`, `minutes`, `hours`, `days`, `months`, `years`.
|
||||
* Значение по умолчанию: `years`.
|
||||
- `minimum_unit` — Опциональный параметр. Минимальная единица измерения для отображения. Более мелкие единицы будут отброшены.
|
||||
* Допустимые значения: `nanoseconds`, `microseconds`, `milliseconds`, `seconds`, `minutes`, `hours`, `days`, `months`, `years`.
|
||||
* Если минимальная единица задана явно и превышает максимальную единицу, будет выкинуто исключение.
|
||||
* Значение по умолчанию: `seconds` если максимальная единица -- секунда или более крупный интервал, в противном случае -- `nanoseconds`.
|
||||
|
||||
Пример:
|
||||
**Пример**
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
|
||||
@ -79,7 +79,7 @@ SELECT * FROM file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 U
|
||||
|
||||
- `*` — заменяет любое количество любых символов кроме `/`, включая отсутствие символов.
|
||||
- `?` — заменяет ровно один любой символ.
|
||||
- `{some_string,another_string,yet_another_one}` — заменяет любую из строк `'some_string', 'another_string', 'yet_another_one'`. В случае, если в какой-либо из строк содержится `/`, то ошибки доступа (permission denied) к существующим, но недоступным директориям/файлам могут быть проигнорированы при помощи настройки [ignore_access_denied_multidirectory_globs](/docs/ru/operations/settings/settings.md#ignore_access_denied_multidirectory_globs).
|
||||
- `{some_string,another_string,yet_another_one}` — заменяет любую из строк `'some_string', 'another_string', 'yet_another_one'`. Эти строки также могут содержать символ `/`.
|
||||
- `{N..M}` — заменяет любое число в интервале от `N` до `M` включительно (может содержать ведущие нули).
|
||||
|
||||
Конструкция с `{}` аналогична табличной функции [remote](remote.md).
|
||||
|
||||
@ -43,7 +43,7 @@ LIMIT 2
|
||||
|
||||
- `*` — Заменяет любое количество любых символов кроме `/`, включая отсутствие символов.
|
||||
- `?` — Заменяет ровно один любой символ.
|
||||
- `{some_string,another_string,yet_another_one}` — Заменяет любую из строк `'some_string', 'another_string', 'yet_another_one'`. В случае, если в какой-либо из строк содержится `/`, то ошибки доступа (permission denied) к существующим, но недоступным директориям/файлам могут быть проигнорированы при помощи настройки [ignore_access_denied_multidirectory_globs](/docs/ru/operations/settings/settings.md#ignore_access_denied_multidirectory_globs).
|
||||
- `{some_string,another_string,yet_another_one}` — Заменяет любую из строк `'some_string', 'another_string', 'yet_another_one'`. Эти строки также могут содержать символ `/`.
|
||||
- `{N..M}` — Заменяет любое число в интервале от `N` до `M` включительно (может содержать ведущие нули).
|
||||
|
||||
Конструкция с `{}` аналогична табличной функции [remote](remote.md).
|
||||
@ -62,5 +62,4 @@ LIMIT 2
|
||||
**Смотрите также**
|
||||
|
||||
- [Виртуальные столбцы](index.md#table_engines-virtual_columns)
|
||||
- Параметр [ignore_access_denied_multidirectory_globs](/docs/ru/operations/settings/settings.md#ignore_access_denied_multidirectory_globs)
|
||||
|
||||
|
||||
@ -270,6 +270,4 @@ ClickHouse成员一旦在您的拉取请求上贴上«可以测试»标签，就
|
||||
|
||||
## 浏览ClickHouse源代码 {#browse-clickhouse-source-code}
|
||||
|
||||
您可以使用 **Woboq** 在线代码浏览器 [点击这里](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示、搜索和索引。 代码快照每天更新。
|
||||
|
||||
此外，您还可以像往常一样浏览源代码 [GitHub](https://github.com/ClickHouse/ClickHouse)
|
||||
|
||||
@ -17,6 +17,13 @@ User=clickhouse
|
||||
Group=clickhouse
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
# The following ClickHouse directives should be used instead of forcing SIGKILL by systemd:
|
||||
# - shutdown_wait_unfinished_queries
|
||||
# - shutdown_wait_unfinished
|
||||
TimeoutStopSec=infinity
|
||||
# Disable forwarding signals by watchdog, since with default systemd's
|
||||
# kill-mode control-group, systemd will send signal to all process in cgroup.
|
||||
Environment=CLICKHOUSE_WATCHDOG_NO_FORWARD=1
|
||||
# Since ClickHouse is systemd aware default 1m30sec may not be enough
|
||||
TimeoutStartSec=0
|
||||
# %p is resolved to the systemd unit name
|
||||
|
||||
@ -132,13 +132,6 @@ func TestConfigFileFrameCopy(t *testing.T) {
|
||||
configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml"))
|
||||
require.Empty(t, errs)
|
||||
i := 0
|
||||
sizes := map[string]int64{
|
||||
"users.xml": int64(2017),
|
||||
"default-password.xml": int64(188),
|
||||
"config.xml": int64(59377),
|
||||
"server-include.xml": int64(168),
|
||||
"user-include.xml": int64(559),
|
||||
}
|
||||
var checkedFiles []string
|
||||
for {
|
||||
values, ok, err := configFrame.Next()
|
||||
@ -153,8 +146,6 @@ func TestConfigFileFrameCopy(t *testing.T) {
|
||||
newPath := path.Join(tmrDir, fileName)
|
||||
err = configFile.Copy(newPath, true)
|
||||
require.FileExists(t, newPath)
|
||||
destInfo, _ := os.Stat(newPath)
|
||||
require.Equal(t, sizes[fileName], destInfo.Size())
|
||||
require.Nil(t, err)
|
||||
bytes, err := ioutil.ReadFile(newPath)
|
||||
require.Nil(t, err)
|
||||
@ -186,13 +177,6 @@ func TestConfigFileFrameCopy(t *testing.T) {
|
||||
configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yaml"))
|
||||
require.Empty(t, errs)
|
||||
i := 0
|
||||
sizes := map[string]int64{
|
||||
"users.yaml": int64(1023),
|
||||
"default-password.yaml": int64(132),
|
||||
"config.yaml": int64(41633),
|
||||
"server-include.yaml": int64(21),
|
||||
"user-include.yaml": int64(120),
|
||||
}
|
||||
var checkedFiles []string
|
||||
for {
|
||||
values, ok, err := configFrame.Next()
|
||||
@ -207,8 +191,6 @@ func TestConfigFileFrameCopy(t *testing.T) {
|
||||
newPath := path.Join(tmrDir, fileName)
|
||||
err = configFile.Copy(newPath, true)
|
||||
require.FileExists(t, newPath)
|
||||
destInfo, _ := os.Stat(newPath)
|
||||
require.Equal(t, sizes[fileName], destInfo.Size())
|
||||
require.Nil(t, err)
|
||||
bytes, err := ioutil.ReadFile(newPath)
|
||||
require.Nil(t, err)
|
||||
|
||||
@ -57,7 +57,7 @@ public:
|
||||
String relative_path_from = validatePathAndGetAsRelative(path_from);
|
||||
String relative_path_to = validatePathAndGetAsRelative(path_to);
|
||||
|
||||
disk_from->copyDirectoryContent(relative_path_from, disk_to, relative_path_to, /* settings= */ {});
|
||||
disk_from->copyDirectoryContent(relative_path_from, disk_to, relative_path_to, /* read_settings= */ {}, /* write_settings= */ {});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@ -32,6 +32,8 @@
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Loggers/Loggers.h>
|
||||
#include <Loggers/OwnFormattingChannel.h>
|
||||
#include <Loggers/OwnPatternFormatter.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <IO/WriteBufferFromFileDescriptor.h>
|
||||
@ -599,7 +601,9 @@ void LocalServer::processConfig()
|
||||
{
|
||||
auto poco_logs_level = Poco::Logger::parseLevel(level);
|
||||
Poco::Logger::root().setLevel(poco_logs_level);
|
||||
Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::SimpleFileChannel>(new Poco::SimpleFileChannel(server_logs_file)));
|
||||
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter;
|
||||
Poco::AutoPtr<OwnFormattingChannel> log = new OwnFormattingChannel(pf, new Poco::SimpleFileChannel(server_logs_file));
|
||||
Poco::Logger::root().setChannel(log);
|
||||
logging_initialized = true;
|
||||
}
|
||||
else if (logging || is_interactive)
|
||||
|
||||
@ -1602,7 +1602,7 @@ try
|
||||
LOG_INFO(log, "Closed all listening sockets.");
|
||||
|
||||
if (current_connections > 0)
|
||||
current_connections = waitServersToFinish(servers_to_start_before_tables, servers_lock, config().getInt("shutdown_wait_unfinished", 5));
|
||||
current_connections = waitServersToFinish(servers_to_start_before_tables, servers_lock, server_settings.shutdown_wait_unfinished);
|
||||
|
||||
if (current_connections)
|
||||
LOG_INFO(log, "Closed connections to servers for tables. But {} remain. Probably some tables of other users cannot finish their connections after context shutdown.", current_connections);
|
||||
@ -1909,7 +1909,7 @@ try
|
||||
global_context->getProcessList().killAllQueries();
|
||||
|
||||
if (current_connections)
|
||||
current_connections = waitServersToFinish(servers, servers_lock, config().getInt("shutdown_wait_unfinished", 5));
|
||||
current_connections = waitServersToFinish(servers, servers_lock, server_settings.shutdown_wait_unfinished);
|
||||
|
||||
if (current_connections)
|
||||
LOG_WARNING(log, "Closed connections. But {} remain."
|
||||
|
||||
@ -1493,4 +1493,10 @@
|
||||
<!-- On Linux systems this can control the behavior of OOM killer.
|
||||
<oom_score>-1000</oom_score>
|
||||
-->
|
||||
|
||||
<!-- Delay (in seconds) to wait for unfinished queries before force exit -->
|
||||
<!-- <shutdown_wait_unfinished>5</shutdown_wait_unfinished> -->
|
||||
|
||||
<!-- If set true ClickHouse will wait for running queries finish before shutdown. -->
|
||||
<!-- <shutdown_wait_unfinished_queries>false</shutdown_wait_unfinished_queries> -->
|
||||
</clickhouse>
|
||||
|
||||
@ -170,6 +170,7 @@ enum class AccessType
|
||||
M(SYSTEM_FETCHES, "SYSTEM STOP FETCHES, SYSTEM START FETCHES, STOP FETCHES, START FETCHES", TABLE, SYSTEM) \
|
||||
M(SYSTEM_MOVES, "SYSTEM STOP MOVES, SYSTEM START MOVES, STOP MOVES, START MOVES", TABLE, SYSTEM) \
|
||||
M(SYSTEM_PULLING_REPLICATION_LOG, "SYSTEM STOP PULLING REPLICATION LOG, SYSTEM START PULLING REPLICATION LOG", TABLE, SYSTEM) \
|
||||
M(SYSTEM_CLEANUP, "SYSTEM STOP CLEANUP, SYSTEM START CLEANUP", TABLE, SYSTEM) \
|
||||
M(SYSTEM_DISTRIBUTED_SENDS, "SYSTEM STOP DISTRIBUTED SENDS, SYSTEM START DISTRIBUTED SENDS, STOP DISTRIBUTED SENDS, START DISTRIBUTED SENDS", TABLE, SYSTEM_SENDS) \
|
||||
M(SYSTEM_REPLICATED_SENDS, "SYSTEM STOP REPLICATED SENDS, SYSTEM START REPLICATED SENDS, STOP REPLICATED SENDS, START REPLICATED SENDS", TABLE, SYSTEM_SENDS) \
|
||||
M(SYSTEM_SENDS, "SYSTEM STOP SENDS, SYSTEM START SENDS, STOP SENDS, START SENDS", GROUP, SYSTEM) \
|
||||
|
||||
@ -556,6 +556,14 @@ bool IAccessStorage::areCredentialsValid(
|
||||
if (credentials.getUserName() != user.getName())
|
||||
return false;
|
||||
|
||||
if (user.valid_until)
|
||||
{
|
||||
const time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
|
||||
|
||||
if (now > user.valid_until)
|
||||
return false;
|
||||
}
|
||||
|
||||
return Authentication::areCredentialsValid(credentials, user.auth_data, external_authenticators);
|
||||
}
|
||||
|
||||
|
||||
@ -340,7 +340,12 @@ bool SettingsConstraints::Checker::check(SettingChange & change,
|
||||
if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value))
|
||||
{
|
||||
if (reaction == THROW_ON_VIOLATION)
|
||||
throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} should not be changed", setting_name);
|
||||
throw Exception(
|
||||
ErrorCodes::SETTING_CONSTRAINT_VIOLATION,
|
||||
"The maximum ({}) value is less than the minimum ({}) value for setting {}",
|
||||
max_value,
|
||||
min_value,
|
||||
setting_name);
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -17,7 +17,8 @@ bool User::equal(const IAccessEntity & other) const
|
||||
const auto & other_user = typeid_cast<const User &>(other);
|
||||
return (auth_data == other_user.auth_data) && (allowed_client_hosts == other_user.allowed_client_hosts)
|
||||
&& (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles)
|
||||
&& (settings == other_user.settings) && (grantees == other_user.grantees) && (default_database == other_user.default_database);
|
||||
&& (settings == other_user.settings) && (grantees == other_user.grantees) && (default_database == other_user.default_database)
|
||||
&& (valid_until == other_user.valid_until);
|
||||
}
|
||||
|
||||
void User::setName(const String & name_)
|
||||
|
||||
@ -23,6 +23,7 @@ struct User : public IAccessEntity
|
||||
SettingsProfileElements settings;
|
||||
RolesOrUsersSet grantees = RolesOrUsersSet::AllTag{};
|
||||
String default_database;
|
||||
time_t valid_until = 0;
|
||||
|
||||
bool equal(const IAccessEntity & other) const override;
|
||||
std::shared_ptr<IAccessEntity> clone() const override { return cloneImpl<User>(); }
|
||||
|
||||
@ -51,7 +51,7 @@ TEST(AccessRights, Union)
|
||||
"CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, UNDROP TABLE, "
|
||||
"TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, "
|
||||
"SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
|
||||
"SYSTEM MOVES, SYSTEM PULLING REPLICATION LOG, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
|
||||
"SYSTEM MOVES, SYSTEM PULLING REPLICATION LOG, SYSTEM CLEANUP, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
|
||||
"SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
|
||||
"SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*, GRANT NAMED COLLECTION ADMIN ON db1");
|
||||
}
|
||||
|
||||
135
src/Analyzer/Passes/AnyFunctionPass.cpp
Normal file
135
src/Analyzer/Passes/AnyFunctionPass.cpp
Normal file
@ -0,0 +1,135 @@
|
||||
#include <Analyzer/Passes/AnyFunctionPass.h>
|
||||
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
|
||||
#include <Analyzer/InDepthQueryTreeVisitor.h>
|
||||
#include <Analyzer/FunctionNode.h>
|
||||
#include <Analyzer/LambdaNode.h>
|
||||
#include <Analyzer/ConstantNode.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class AnyFunctionVisitor : public InDepthQueryTreeVisitorWithContext<AnyFunctionVisitor>
|
||||
{
|
||||
private:
|
||||
bool canRewrite(const FunctionNode * function_node)
|
||||
{
|
||||
for (const auto & argument : function_node->getArguments().getNodes())
|
||||
{
|
||||
if (argument->as<LambdaNode>())
|
||||
return false;
|
||||
|
||||
if (const auto * inside_function = argument->as<FunctionNode>())
|
||||
{
|
||||
/// Function arrayJoin is special and should be skipped (think about it as
|
||||
/// an aggregate function), otherwise wrong result will be produced.
|
||||
/// For example:
|
||||
/// SELECT *, any(arrayJoin([[], []])) FROM numbers(1) GROUP BY number
|
||||
/// ┌─number─┬─arrayJoin(array(array(), array()))─┐
|
||||
/// │ 0 │ [] │
|
||||
/// │ 0 │ [] │
|
||||
/// └────────┴────────────────────────────────────┘
|
||||
if (inside_function->getFunctionName() == "arrayJoin")
|
||||
return false;
|
||||
|
||||
if (!canRewrite(inside_function))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public:
|
||||
using Base = InDepthQueryTreeVisitorWithContext<AnyFunctionVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
void enterImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_move_functions_out_of_any)
|
||||
return;
|
||||
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
if (!function_node)
|
||||
return;
|
||||
|
||||
/// check function is any
|
||||
const auto & function_name = function_node->getFunctionName();
|
||||
if (!(function_name == "any" || function_name == "anyLast"))
|
||||
return;
|
||||
|
||||
auto & arguments = function_node->getArguments().getNodes();
|
||||
if (arguments.size() != 1)
|
||||
return;
|
||||
|
||||
auto * inside_function_node = arguments[0]->as<FunctionNode>();
|
||||
|
||||
/// check argument is a function
|
||||
if (!inside_function_node)
|
||||
return;
|
||||
|
||||
/// check arguments can not contain arrayJoin or lambda
|
||||
if (!canRewrite(inside_function_node))
|
||||
return;
|
||||
|
||||
auto & inside_arguments = inside_function_node->getArguments().getNodes();
|
||||
|
||||
/// case any(f())
|
||||
if (inside_arguments.empty())
|
||||
return;
|
||||
|
||||
if (rewritten.contains(node.get()))
|
||||
{
|
||||
node = rewritten.at(node.get());
|
||||
return;
|
||||
}
|
||||
|
||||
/// checking done, rewrite function
|
||||
bool pushed = false;
|
||||
for (auto & inside_argument : inside_arguments)
|
||||
{
|
||||
if (inside_argument->as<ConstantNode>()) /// skip constant node
|
||||
break;
|
||||
|
||||
AggregateFunctionProperties properties;
|
||||
auto aggregate_function = AggregateFunctionFactory::instance().get(function_name, {inside_argument->getResultType()}, {}, properties);
|
||||
|
||||
auto any_function = std::make_shared<FunctionNode>(function_name);
|
||||
any_function->resolveAsAggregateFunction(std::move(aggregate_function));
|
||||
|
||||
auto & any_function_arguments = any_function->getArguments().getNodes();
|
||||
any_function_arguments.push_back(std::move(inside_argument));
|
||||
|
||||
inside_argument = std::move(any_function);
|
||||
pushed = true;
|
||||
}
|
||||
|
||||
if (pushed)
|
||||
{
|
||||
rewritten.insert({node.get(), arguments[0]});
|
||||
node = arguments[0];
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
/// After query analysis alias will be rewritten to QueryTreeNode
|
||||
/// whose memory address is same with the original one.
|
||||
/// So we can reuse the rewritten one.
|
||||
std::unordered_map<IQueryTreeNode *, QueryTreeNodePtr > rewritten;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void AnyFunctionPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
AnyFunctionVisitor visitor(context);
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
}
|
||||
27
src/Analyzer/Passes/AnyFunctionPass.h
Normal file
27
src/Analyzer/Passes/AnyFunctionPass.h
Normal file
@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
#include <Analyzer/IQueryTreePass.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Rewrite 'any' and 'anyLast' functions pushing them inside original function.
|
||||
*
|
||||
* Example: any(f(x, y, g(z)))
|
||||
* Result: f(any(x), any(y), g(any(z)))
|
||||
*/
|
||||
class AnyFunctionPass final : public IQueryTreePass
|
||||
{
|
||||
public:
|
||||
String getName() override { return "AnyFunction"; }
|
||||
|
||||
String getDescription() override
|
||||
{
|
||||
return "Rewrite 'any' and 'anyLast' functions pushing them inside original function.";
|
||||
}
|
||||
|
||||
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
@ -6247,7 +6247,7 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
|
||||
.getTable(insertion_table, scope_context)
|
||||
->getInMemoryMetadataPtr()
|
||||
->getColumns();
|
||||
const auto & insert_column_names = scope_context->hasInsertionTableColumnNames() ? *scope_context->getInsertionTableColumnNames() : insert_columns.getInsertable().getNames();
|
||||
const auto & insert_column_names = scope_context->hasInsertionTableColumnNames() ? *scope_context->getInsertionTableColumnNames() : insert_columns.getOrdinary().getNames();
|
||||
DB::ColumnsDescription structure_hint;
|
||||
|
||||
bool use_columns_from_insert_query = true;
|
||||
@ -6282,6 +6282,8 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
|
||||
|
||||
ColumnDescription column = insert_columns.get(*insert_column_name_it);
|
||||
column.name = identifier_node->getIdentifier().getFullName();
|
||||
/// Change ephemeral columns to default columns.
|
||||
column.default_desc.kind = ColumnDefaultKind::Default;
|
||||
structure_hint.add(std::move(column));
|
||||
}
|
||||
|
||||
@ -6356,8 +6358,13 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
|
||||
if (asterisk)
|
||||
{
|
||||
for (; insert_column_name_it != insert_column_names_end; ++insert_column_name_it)
|
||||
{
|
||||
ColumnDescription column = insert_columns.get(*insert_column_name_it);
|
||||
/// Change ephemeral columns to default columns.
|
||||
column.default_desc.kind = ColumnDefaultKind::Default;
|
||||
structure_hint.add(insert_columns.get(*insert_column_name_it));
|
||||
}
|
||||
}
|
||||
|
||||
if (!structure_hint.empty())
|
||||
table_function_ptr->setStructureHint(structure_hint);
|
||||
|
||||
@ -43,8 +43,10 @@
|
||||
#include <Analyzer/Passes/CrossToInnerJoinPass.h>
|
||||
#include <Analyzer/Passes/ShardNumColumnToFunctionPass.h>
|
||||
#include <Analyzer/Passes/ConvertQueryToCNFPass.h>
|
||||
#include <Analyzer/Passes/AnyFunctionPass.h>
|
||||
#include <Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -281,7 +283,10 @@ void addQueryTreePasses(QueryTreePassManager & manager)
|
||||
manager.addPass(std::make_unique<AutoFinalOnQueryPass>());
|
||||
manager.addPass(std::make_unique<CrossToInnerJoinPass>());
|
||||
manager.addPass(std::make_unique<ShardNumColumnToFunctionPass>());
|
||||
|
||||
manager.addPass(std::make_unique<AnyFunctionPass>());
|
||||
manager.addPass(std::make_unique<OptimizeDateOrDateTimeConverterWithPreimagePass>());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -46,7 +46,7 @@ void BackupReaderDisk::copyFileToDisk(const String & path_in_backup, size_t file
|
||||
{
|
||||
/// Use more optimal way.
|
||||
LOG_TRACE(log, "Copying file {} from disk {} to disk {}", path_in_backup, disk->getName(), destination_disk->getName());
|
||||
disk->copyFile(root_path / path_in_backup, *destination_disk, destination_path, write_settings);
|
||||
disk->copyFile(root_path / path_in_backup, *destination_disk, destination_path, read_settings, write_settings);
|
||||
return; /// copied!
|
||||
}
|
||||
}
|
||||
@ -119,7 +119,7 @@ void BackupWriterDisk::copyFileFromDisk(const String & path_in_backup, DiskPtr s
|
||||
LOG_TRACE(log, "Copying file {} from disk {} to disk {}", src_path, src_disk->getName(), disk->getName());
|
||||
auto dest_file_path = root_path / path_in_backup;
|
||||
disk->createDirectories(dest_file_path.parent_path());
|
||||
src_disk->copyFile(src_path, *disk, dest_file_path, write_settings);
|
||||
src_disk->copyFile(src_path, *disk, dest_file_path, read_settings, write_settings);
|
||||
return; /// copied!
|
||||
}
|
||||
}
|
||||
|
||||
@ -170,6 +170,7 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s
|
||||
/* dest_bucket= */ blob_path[1],
|
||||
/* dest_key= */ blob_path[0],
|
||||
request_settings,
|
||||
read_settings,
|
||||
object_attributes,
|
||||
threadPoolCallbackRunner<void>(getBackupsIOThreadPool().get(), "BackupReaderS3"),
|
||||
/* for_disk_s3= */ true);
|
||||
@ -230,6 +231,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src
|
||||
s3_uri.bucket,
|
||||
fs::path(s3_uri.key) / path_in_backup,
|
||||
request_settings,
|
||||
read_settings,
|
||||
{},
|
||||
threadPoolCallbackRunner<void>(getBackupsIOThreadPool().get(), "BackupWriterS3"));
|
||||
return; /// copied!
|
||||
|
||||
@ -218,20 +218,20 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
BackupsWorker::BackupsWorker(size_t num_backup_threads, size_t num_restore_threads, bool allow_concurrent_backups_, bool allow_concurrent_restores_)
|
||||
BackupsWorker::BackupsWorker(ContextPtr global_context, size_t num_backup_threads, size_t num_restore_threads, bool allow_concurrent_backups_, bool allow_concurrent_restores_)
|
||||
: backups_thread_pool(std::make_unique<ThreadPool>(CurrentMetrics::BackupsThreads, CurrentMetrics::BackupsThreadsActive, num_backup_threads, /* max_free_threads = */ 0, num_backup_threads))
|
||||
, restores_thread_pool(std::make_unique<ThreadPool>(CurrentMetrics::RestoreThreads, CurrentMetrics::RestoreThreadsActive, num_restore_threads, /* max_free_threads = */ 0, num_restore_threads))
|
||||
, log(&Poco::Logger::get("BackupsWorker"))
|
||||
, allow_concurrent_backups(allow_concurrent_backups_)
|
||||
, allow_concurrent_restores(allow_concurrent_restores_)
|
||||
{
|
||||
backup_log = global_context->getBackupLog();
|
||||
/// We set max_free_threads = 0 because we don't want to keep any threads if there is no BACKUP or RESTORE query running right now.
|
||||
}
|
||||
|
||||
|
||||
OperationID BackupsWorker::start(const ASTPtr & backup_or_restore_query, ContextMutablePtr context)
|
||||
{
|
||||
backup_log = context->getBackupLog();
|
||||
const ASTBackupQuery & backup_query = typeid_cast<const ASTBackupQuery &>(*backup_or_restore_query);
|
||||
if (backup_query.kind == ASTBackupQuery::Kind::BACKUP)
|
||||
return startMakingBackup(backup_or_restore_query, context);
|
||||
|
||||
@ -32,7 +32,7 @@ class BackupLog;
|
||||
class BackupsWorker
|
||||
{
|
||||
public:
|
||||
BackupsWorker(size_t num_backup_threads, size_t num_restore_threads, bool allow_concurrent_backups_, bool allow_concurrent_restores_);
|
||||
BackupsWorker(ContextPtr global_context, size_t num_backup_threads, size_t num_restore_threads, bool allow_concurrent_backups_, bool allow_concurrent_restores_);
|
||||
|
||||
/// Waits until all tasks have been completed.
|
||||
void shutdown();
|
||||
|
||||
@ -140,7 +140,7 @@ void IBridge::initialize(Application & self)
|
||||
throw Poco::OpenFileException("Cannot attach stdout to " + stdout_path);
|
||||
|
||||
/// Disable buffering for stdout.
|
||||
setbuf(stdout, nullptr);
|
||||
setbuf(stdout, nullptr); // NOLINT(cert-msc24-c,cert-msc33-c)
|
||||
}
|
||||
const auto stderr_path = config().getString("logger.stderr", "");
|
||||
if (!stderr_path.empty())
|
||||
@ -149,7 +149,7 @@ void IBridge::initialize(Application & self)
|
||||
throw Poco::OpenFileException("Cannot attach stderr to " + stderr_path);
|
||||
|
||||
/// Disable buffering for stderr.
|
||||
setbuf(stderr, nullptr);
|
||||
setbuf(stderr, nullptr); // NOLINT(cert-msc24-c,cert-msc33-c)
|
||||
}
|
||||
|
||||
buildLoggers(config(), logger(), self.commandName());
|
||||
|
||||
@ -1475,13 +1475,23 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
|
||||
current_format = FormatFactory::instance().getFormatFromFileName(in_file, true);
|
||||
|
||||
/// Create temporary storage file, to support globs and parallel reading
|
||||
/// StorageFile doesn't support ephemeral/materialized/alias columns.
|
||||
/// We should change ephemeral columns to ordinary and ignore materialized/alias columns.
|
||||
ColumnsDescription columns_for_storage_file;
|
||||
for (const auto & [name, _] : columns_description_for_query.getInsertable())
|
||||
{
|
||||
ColumnDescription column = columns_description_for_query.get(name);
|
||||
column.default_desc.kind = ColumnDefaultKind::Default;
|
||||
columns_for_storage_file.add(std::move(column));
|
||||
}
|
||||
|
||||
StorageFile::CommonArguments args{
|
||||
WithContext(global_context),
|
||||
parsed_insert_query->table_id,
|
||||
current_format,
|
||||
getFormatSettings(global_context),
|
||||
compression_method,
|
||||
columns_description_for_query,
|
||||
columns_for_storage_file,
|
||||
ConstraintsDescription{},
|
||||
String{},
|
||||
{},
|
||||
|
||||
@ -115,7 +115,7 @@ replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String &
|
||||
return replxx::Replxx::completions_t(range.first, range.second);
|
||||
}
|
||||
|
||||
void LineReader::Suggest::addWords(Words && new_words)
|
||||
void LineReader::Suggest::addWords(Words && new_words) // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved)
|
||||
{
|
||||
Words new_words_no_case = new_words;
|
||||
if (!new_words.empty())
|
||||
|
||||
@ -310,8 +310,8 @@ static String cacheElemToString(const Poco::Net::IPAddress & addr) { return addr
|
||||
|
||||
template <typename UpdateF, typename ElemsT>
|
||||
bool DNSResolver::updateCacheImpl(
|
||||
UpdateF && update_func,
|
||||
ElemsT && elems,
|
||||
UpdateF && update_func, // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
ElemsT && elems, // NOLINT(cppcoreguidelines-missing-std-forward)
|
||||
UInt32 max_consecutive_failures,
|
||||
FormatStringHelper<String> notfound_log_msg,
|
||||
FormatStringHelper<String> dropped_log_msg)
|
||||
|
||||
@ -800,7 +800,7 @@ Dwarf::CompilationUnit Dwarf::getCompilationUnit(uint64_t offset) const
|
||||
die,
|
||||
[&](const Attribute & attr)
|
||||
{
|
||||
switch (attr.spec.name)
|
||||
switch (attr.spec.name) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case DW_AT_addr_base:
|
||||
case DW_AT_GNU_addr_base:
|
||||
@ -996,7 +996,7 @@ bool Dwarf::findLocation(
|
||||
|
||||
forEachAttribute(cu, die, [&](const Attribute & attr)
|
||||
{
|
||||
switch (attr.spec.name)
|
||||
switch (attr.spec.name) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case DW_AT_stmt_list:
|
||||
// Offset in .debug_line for the line number VM program for this
|
||||
@ -1143,7 +1143,7 @@ void Dwarf::findSubProgramDieForAddress(const CompilationUnit & cu,
|
||||
std::optional<uint64_t> range_offset;
|
||||
forEachAttribute(cu, child_die, [&](const Attribute & attr)
|
||||
{
|
||||
switch (attr.spec.name)
|
||||
switch (attr.spec.name) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case DW_AT_ranges:
|
||||
range_offset = std::get<uint64_t>(attr.attr_value);
|
||||
@ -1234,7 +1234,7 @@ void Dwarf::findInlinedSubroutineDieForAddress(
|
||||
std::optional<uint64_t> range_offset;
|
||||
forEachAttribute(cu, child_die, [&](const Attribute & attr)
|
||||
{
|
||||
switch (attr.spec.name)
|
||||
switch (attr.spec.name) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case DW_AT_ranges:
|
||||
range_offset = std::get<uint64_t>(attr.attr_value);
|
||||
@ -1349,7 +1349,7 @@ void Dwarf::findInlinedSubroutineDieForAddress(
|
||||
// its DW_AT_call_file and DW_AT_call_line.
|
||||
forEachAttribute(srcu, die_to_look_for_name, [&](const Attribute & attr)
|
||||
{
|
||||
switch (attr.spec.name)
|
||||
switch (attr.spec.name) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case DW_AT_linkage_name:
|
||||
name = std::get<std::string_view>(attr.attr_value);
|
||||
@ -1910,7 +1910,7 @@ Dwarf::LineNumberVM::FileName Dwarf::LineNumberVM::getFileName(uint64_t index) c
|
||||
auto attr = readLineNumberAttribute(is64Bit_, format, file_names, debugStr_, debugLineStr_);
|
||||
if (i == index)
|
||||
{
|
||||
switch (attr.content_type_code)
|
||||
switch (attr.content_type_code) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case DW_LNCT_path:
|
||||
fn.relativeName = std::get<std::string_view>(attr.attr_value);
|
||||
@ -2055,7 +2055,7 @@ Dwarf::LineNumberVM::StepResult Dwarf::LineNumberVM::step(std::string_view & pro
|
||||
{ // standard opcode
|
||||
// Only interpret opcodes that are recognized by the version we're parsing;
|
||||
// the others are vendor extensions and we should ignore them.
|
||||
switch (opcode)
|
||||
switch (opcode) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case DW_LNS_copy:
|
||||
basicBlock_ = false;
|
||||
@ -2127,7 +2127,7 @@ Dwarf::LineNumberVM::StepResult Dwarf::LineNumberVM::step(std::string_view & pro
|
||||
auto extended_opcode = read<uint8_t>(program);
|
||||
--length;
|
||||
|
||||
switch (extended_opcode)
|
||||
switch (extended_opcode) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case DW_LNE_end_sequence:
|
||||
return END;
|
||||
|
||||
@ -585,7 +585,9 @@
|
||||
M(700, USER_SESSION_LIMIT_EXCEEDED) \
|
||||
M(701, CLUSTER_DOESNT_EXIST) \
|
||||
M(702, CLIENT_INFO_DOES_NOT_MATCH) \
|
||||
\
|
||||
M(703, INVALID_IDENTIFIER) \
|
||||
M(704, CANNOT_USE_QUERY_CACHE_WITH_NONDETERMINISTIC_FUNCTIONS) \
|
||||
M(705, TABLE_NOT_EMPTY) \
|
||||
M(999, KEEPER_EXCEPTION) \
|
||||
M(1000, POCO_EXCEPTION) \
|
||||
M(1001, STD_EXCEPTION) \
|
||||
|
||||
@ -310,7 +310,10 @@ requires (sizeof(T) <= sizeof(UInt64))
|
||||
inline size_t hashCRC32(T key, DB::UInt64 updated_value = -1)
|
||||
{
|
||||
DB::UInt64 out {0};
|
||||
if constexpr (std::endian::native == std::endian::little)
|
||||
std::memcpy(&out, &key, sizeof(T));
|
||||
else
|
||||
std::memcpy(reinterpret_cast<char*>(&out) + sizeof(UInt64) - sizeof(T), &key, sizeof(T));
|
||||
return intHashCRC32(out, updated_value);
|
||||
}
|
||||
|
||||
|
||||
@ -496,6 +496,14 @@ OptimizedRegularExpression::OptimizedRegularExpression(const std::string & regex
|
||||
regexp_options.set_dot_nl(true);
|
||||
|
||||
re2 = std::make_unique<re2::RE2>(regexp_, regexp_options);
|
||||
|
||||
/// Fallback to latin1 to allow matching binary data.
|
||||
if (!re2->ok() && re2->error_code() == re2::RE2::ErrorCode::ErrorBadUTF8)
|
||||
{
|
||||
regexp_options.set_encoding(re2::RE2::Options::EncodingLatin1);
|
||||
re2 = std::make_unique<re2::RE2>(regexp_, regexp_options);
|
||||
}
|
||||
|
||||
if (!re2->ok())
|
||||
{
|
||||
throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP,
|
||||
|
||||
@ -32,6 +32,12 @@ TimerDescriptor::TimerDescriptor(TimerDescriptor && other) noexcept : timer_fd(o
|
||||
other.timer_fd = -1;
|
||||
}
|
||||
|
||||
TimerDescriptor & TimerDescriptor::operator=(DB::TimerDescriptor && other) noexcept
|
||||
{
|
||||
std::swap(timer_fd, other.timer_fd);
|
||||
return *this;
|
||||
}
|
||||
|
||||
TimerDescriptor::~TimerDescriptor()
|
||||
{
|
||||
/// Do not check for result cause cannot throw exception.
|
||||
|
||||
@ -18,7 +18,7 @@ public:
|
||||
TimerDescriptor(const TimerDescriptor &) = delete;
|
||||
TimerDescriptor & operator=(const TimerDescriptor &) = delete;
|
||||
TimerDescriptor(TimerDescriptor && other) noexcept;
|
||||
TimerDescriptor & operator=(TimerDescriptor &&) = default;
|
||||
TimerDescriptor & operator=(TimerDescriptor &&) noexcept;
|
||||
|
||||
int getDescriptor() const { return timer_fd; }
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/ZooKeeper/IKeeper.h>
|
||||
#include <Common/thread_local_rng.h>
|
||||
#include <random>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -54,6 +56,34 @@ Exception::Exception(const Error code_)
|
||||
Exception::Exception(const Exception & exc) = default;
|
||||
|
||||
|
||||
SimpleFaultInjection::SimpleFaultInjection(Float64 probability_before, Float64 probability_after_, const String & description_)
|
||||
{
|
||||
if (likely(probability_before == 0.0) && likely(probability_after_ == 0.0))
|
||||
return;
|
||||
|
||||
std::bernoulli_distribution fault(probability_before);
|
||||
if (fault(thread_local_rng))
|
||||
throw Coordination::Exception(Coordination::Error::ZCONNECTIONLOSS, "Fault injected (before {})", description_);
|
||||
|
||||
probability_after = probability_after_;
|
||||
description = description_;
|
||||
exceptions_level = std::uncaught_exceptions();
|
||||
}
|
||||
|
||||
SimpleFaultInjection::~SimpleFaultInjection() noexcept(false)
|
||||
{
|
||||
if (likely(probability_after == 0.0))
|
||||
return;
|
||||
|
||||
/// Do not throw from dtor during unwinding
|
||||
if (exceptions_level != std::uncaught_exceptions())
|
||||
return;
|
||||
|
||||
std::bernoulli_distribution fault(probability_after);
|
||||
if (fault(thread_local_rng))
|
||||
throw Coordination::Exception(Coordination::Error::ZCONNECTIONLOSS, "Fault injected (after {})", description);
|
||||
}
|
||||
|
||||
using namespace DB;
|
||||
|
||||
|
||||
|
||||
@ -510,6 +510,18 @@ public:
|
||||
const Error code;
|
||||
};
|
||||
|
||||
class SimpleFaultInjection
|
||||
{
|
||||
public:
|
||||
SimpleFaultInjection(Float64 probability_before, Float64 probability_after_, const String & description_);
|
||||
~SimpleFaultInjection() noexcept(false);
|
||||
|
||||
private:
|
||||
Float64 probability_after = 0;
|
||||
String description;
|
||||
int exceptions_level = 0;
|
||||
};
|
||||
|
||||
|
||||
/** Usage scenario:
|
||||
* - create an object and issue commands;
|
||||
|
||||
@ -36,9 +36,7 @@ using namespace DB;
|
||||
namespace
|
||||
{
|
||||
|
||||
template <class T> using is_pod = std::is_trivial<std::is_standard_layout<T>>;
|
||||
template <class T> inline constexpr bool is_pod_v = is_pod<T>::value;
|
||||
|
||||
template <class T> inline constexpr bool is_pod_v = std::is_trivial_v<std::is_standard_layout<T>>;
|
||||
|
||||
template <typename T>
|
||||
struct AsHexStringHelper
|
||||
|
||||
@ -38,7 +38,7 @@ struct Settings;
|
||||
M(UInt64, stale_log_gap, 10000, "When node became stale and should receive snapshots from leader", 0) \
|
||||
M(UInt64, fresh_log_gap, 200, "When node became fresh", 0) \
|
||||
M(UInt64, max_request_queue_size, 100000, "Maximum number of request that can be in queue for processing", 0) \
|
||||
M(UInt64, max_requests_batch_size, 1000, "Max size of batch of requests that can be sent to RAFT", 0) \
|
||||
M(UInt64, max_requests_batch_size, 100, "Max size of batch of requests that can be sent to RAFT", 0) \
|
||||
M(UInt64, max_requests_batch_bytes_size, 100*1024, "Max size in bytes of batch of requests that can be sent to RAFT", 0) \
|
||||
M(UInt64, max_flush_batch_size, 1000, "Max size of batch of requests that can be flushed together", 0) \
|
||||
M(UInt64, max_requests_quick_batch_size, 100, "Max size of batch of requests to try to get before proceeding with RAFT. Keeper will not wait for requests but take only requests that are already in queue" , 0) \
|
||||
|
||||
@ -16,6 +16,7 @@ namespace DB
|
||||
#define SERVER_SETTINGS(M, ALIAS) \
|
||||
M(Bool, show_addresses_in_stack_traces, true, "If it is set true will show addresses in stack traces", 0) \
|
||||
M(Bool, shutdown_wait_unfinished_queries, false, "If set true ClickHouse will wait for running queries finish before shutdown.", 0) \
|
||||
M(UInt64, shutdown_wait_unfinished, 5, "Delay in seconds to wait for unfinished queries", 0) \
|
||||
M(UInt64, max_thread_pool_size, 10000, "The maximum number of threads that could be allocated from the OS and used for query execution and background operations.", 0) \
|
||||
M(UInt64, max_thread_pool_free_size, 1000, "The maximum number of threads that will always stay in a global thread pool once allocated and remain idle in case of insufficient number of tasks.", 0) \
|
||||
M(UInt64, thread_pool_queue_size, 10000, "The maximum number of tasks that will be placed in a queue and wait for execution.", 0) \
|
||||
|
||||
@ -105,6 +105,7 @@ class IColumn;
|
||||
M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \
|
||||
M(UInt64, s3_retry_attempts, 10, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \
|
||||
M(UInt64, s3_request_timeout_ms, 3000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \
|
||||
M(UInt64, s3_http_connection_pool_size, 1000, "How many reusable open connections to keep per S3 endpoint. Only applies to the S3 table engine and table function, not to S3 disks (for disks, use disk config instead). Global setting, can only be set in config, overriding it per session or per query has no effect.", 0) \
|
||||
M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \
|
||||
M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \
|
||||
M(UInt64, hdfs_replication, 0, "The actual number of replications can be specified when the hdfs file is created.", 0) \
|
||||
@ -502,6 +503,7 @@ class IColumn;
|
||||
M(Bool, reject_expensive_hyperscan_regexps, true, "Reject patterns which will likely be expensive to evaluate with hyperscan (due to NFA state explosion)", 0) \
|
||||
M(Bool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \
|
||||
M(Bool, allow_introspection_functions, false, "Allow functions for introspection of ELF and DWARF for query profiling. These functions are slow and may impose security considerations.", 0) \
|
||||
M(Bool, splitby_max_substrings_includes_remaining_string, false, "Functions 'splitBy*()' with 'max_substrings' argument > 0 include the remaining string as last element in the result", 0) \
|
||||
\
|
||||
M(Bool, allow_execute_multiif_columnar, true, "Allow execute multiIf function columnar", 0) \
|
||||
M(Bool, formatdatetime_f_prints_single_zero, false, "Formatter '%f' in function 'formatDateTime()' produces a single zero instead of six zeros if the formatted value has no fractional seconds.", 0) \
|
||||
@ -803,7 +805,6 @@ class IColumn;
|
||||
M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \
|
||||
M(Bool, allow_create_index_without_type, false, "Allow CREATE INDEX query without TYPE. Query will be ignored. Made for SQL compatibility tests.", 0)\
|
||||
M(Bool, create_index_ignore_unique, false, "Ignore UNIQUE keyword in CREATE UNIQUE INDEX. Made for SQL compatibility tests.", 0) \
|
||||
M(Bool, ignore_access_denied_multidirectory_globs, false, "Ignore access denied errors when processing multi-directory globs for file & HDFS.", 0)\
|
||||
|
||||
// End of COMMON_SETTINGS
|
||||
// Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS.
|
||||
|
||||
@ -429,7 +429,7 @@ constexpr auto getEnumValues()
|
||||
if (it != map.end()) \
|
||||
return it->second; \
|
||||
throw Exception(ERROR_CODE_FOR_UNEXPECTED_NAME, \
|
||||
"Unexpected value of " #NEW_NAME ":{}", std::to_string(std::underlying_type<EnumType>::type(value))); \
|
||||
"Unexpected value of " #NEW_NAME ":{}", std::to_string(std::underlying_type_t<EnumType>(value))); \
|
||||
} \
|
||||
\
|
||||
typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \
|
||||
|
||||
@ -103,6 +103,7 @@ static const size_t signal_pipe_buf_size =
|
||||
+ sizeof(siginfo_t)
|
||||
+ sizeof(ucontext_t*)
|
||||
+ sizeof(StackTrace)
|
||||
+ sizeof(UInt64)
|
||||
+ sizeof(UInt32)
|
||||
+ sizeof(void*);
|
||||
|
||||
@ -181,6 +182,15 @@ static void signalHandler(int sig, siginfo_t * info, void * context)
|
||||
errno = saved_errno;
|
||||
}
|
||||
|
||||
static bool getenvBool(const char * name)
|
||||
{
|
||||
bool res = false;
|
||||
const char * env_var = getenv(name); // NOLINT(concurrency-mt-unsafe)
|
||||
if (env_var && 0 == strcmp(env_var, "1"))
|
||||
res = true;
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
/// Avoid link time dependency on DB/Interpreters - will use this function only when linked.
|
||||
__attribute__((__weak__)) void collectCrashLog(
|
||||
@ -843,7 +853,7 @@ void BaseDaemon::initialize(Application & self)
|
||||
throw Poco::OpenFileException("Cannot attach stderr to " + stderr_path);
|
||||
|
||||
/// Disable buffering for stderr
|
||||
setbuf(stderr, nullptr);
|
||||
setbuf(stderr, nullptr); // NOLINT(cert-msc24-c,cert-msc33-c)
|
||||
}
|
||||
|
||||
if ((!log_path.empty() && is_daemon) || config().has("logger.stdout"))
|
||||
@ -1110,10 +1120,8 @@ void BaseDaemon::setupWatchdog()
|
||||
if (argv0)
|
||||
original_process_name = argv0;
|
||||
|
||||
bool restart = false;
|
||||
const char * env_watchdog_restart = getenv("CLICKHOUSE_WATCHDOG_RESTART"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (env_watchdog_restart && 0 == strcmp(env_watchdog_restart, "1"))
|
||||
restart = true;
|
||||
bool restart = getenvBool("CLICKHOUSE_WATCHDOG_RESTART");
|
||||
bool forward_signals = !getenvBool("CLICKHOUSE_WATCHDOG_NO_FORWARD");
|
||||
|
||||
while (true)
|
||||
{
|
||||
@ -1194,6 +1202,8 @@ void BaseDaemon::setupWatchdog()
|
||||
logger().information(fmt::format("Will watch for the process with pid {}", pid));
|
||||
|
||||
/// Forward signals to the child process.
|
||||
if (forward_signals)
|
||||
{
|
||||
addSignalHandler(
|
||||
{SIGHUP, SIGINT, SIGQUIT, SIGTERM},
|
||||
[](int sig, siginfo_t *, void *)
|
||||
@ -1211,6 +1221,18 @@ void BaseDaemon::setupWatchdog()
|
||||
}
|
||||
},
|
||||
nullptr);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (const auto & sig : {SIGHUP, SIGINT, SIGQUIT, SIGTERM})
|
||||
{
|
||||
if (SIG_ERR == signal(sig, SIG_IGN))
|
||||
{
|
||||
char * signal_description = strsignal(sig); // NOLINT(concurrency-mt-unsafe)
|
||||
throwFromErrno(fmt::format("Cannot ignore {}", signal_description), ErrorCodes::SYSTEM_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int status = 0;
|
||||
do
|
||||
|
||||
@ -111,7 +111,7 @@ MutableColumnUniquePtr DataTypeLowCardinality::createColumnUnique(const IDataTyp
|
||||
{
|
||||
auto creator = [&](auto x)
|
||||
{
|
||||
using ColumnType = typename std::remove_pointer<decltype(x)>::type;
|
||||
using ColumnType = typename std::remove_pointer_t<decltype(x)>;
|
||||
return ColumnUnique<ColumnType>::create(keys_type);
|
||||
};
|
||||
return createColumnUniqueImpl(keys_type, creator);
|
||||
@ -121,7 +121,7 @@ MutableColumnUniquePtr DataTypeLowCardinality::createColumnUnique(const IDataTyp
|
||||
{
|
||||
auto creator = [&](auto x)
|
||||
{
|
||||
using ColumnType = typename std::remove_pointer<decltype(x)>::type;
|
||||
using ColumnType = typename std::remove_pointer_t<decltype(x)>;
|
||||
return ColumnUnique<ColumnType>::create(std::move(keys), keys_type.isNullable());
|
||||
};
|
||||
return createColumnUniqueImpl(keys_type, creator);
|
||||
|
||||
@ -35,6 +35,12 @@ static DataTypePtr createNumericDataType(const ASTPtr & arguments)
|
||||
return std::make_shared<DataTypeNumber<T>>();
|
||||
}
|
||||
|
||||
bool isUInt64ThatCanBeInt64(const DataTypePtr & type)
|
||||
{
|
||||
const DataTypeUInt64 * uint64_type = typeid_cast<const DataTypeUInt64 *>(type.get());
|
||||
return uint64_type && uint64_type->canUnsignedBeSigned();
|
||||
}
|
||||
|
||||
|
||||
void registerDataTypeNumbers(DataTypeFactory & factory)
|
||||
{
|
||||
|
||||
@ -9,10 +9,17 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
class DataTypeNumber final : public DataTypeNumberBase<T>
|
||||
{
|
||||
public:
|
||||
DataTypeNumber() = default;
|
||||
|
||||
bool equals(const IDataType & rhs) const override { return typeid(rhs) == typeid(*this); }
|
||||
|
||||
bool canBeUsedAsVersion() const override { return true; }
|
||||
@ -32,6 +39,20 @@ public:
|
||||
{
|
||||
return std::make_shared<SerializationNumber<T>>();
|
||||
}
|
||||
|
||||
/// Special constructor for unsigned integers that can also fit into signed integer.
|
||||
/// It's used for better type inference from fields.
|
||||
/// See getLeastSupertype.cpp::convertUInt64toInt64IfPossible and FieldToDataType.cpp
|
||||
explicit DataTypeNumber(bool unsigned_can_be_signed_) : DataTypeNumberBase<T>(), unsigned_can_be_signed(unsigned_can_be_signed_)
|
||||
{
|
||||
if constexpr (std::is_signed_v<T>)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "DataTypeNumber constructor with bool argument should not be used with signed integers");
|
||||
}
|
||||
|
||||
bool canUnsignedBeSigned() const { return unsigned_can_be_signed; }
|
||||
|
||||
private:
|
||||
bool unsigned_can_be_signed = false;
|
||||
};
|
||||
|
||||
using DataTypeUInt8 = DataTypeNumber<UInt8>;
|
||||
@ -50,4 +71,6 @@ using DataTypeInt128 = DataTypeNumber<Int128>;
|
||||
using DataTypeUInt256 = DataTypeNumber<UInt256>;
|
||||
using DataTypeInt256 = DataTypeNumber<Int256>;
|
||||
|
||||
bool isUInt64ThatCanBeInt64(const DataTypePtr & type);
|
||||
|
||||
}
|
||||
|
||||
@ -36,6 +36,7 @@ DataTypePtr FieldToDataType<on_error>::operator() (const UInt64 & x) const
|
||||
if (x <= std::numeric_limits<UInt8>::max()) return std::make_shared<DataTypeUInt8>();
|
||||
if (x <= std::numeric_limits<UInt16>::max()) return std::make_shared<DataTypeUInt16>();
|
||||
if (x <= std::numeric_limits<UInt32>::max()) return std::make_shared<DataTypeUInt32>();
|
||||
if (x <= std::numeric_limits<Int64>::max()) return std::make_shared<DataTypeUInt64>(/*unsigned_can_be_signed=*/true);
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
@ -136,17 +137,8 @@ DataTypePtr FieldToDataType<on_error>::operator() (const Array & x) const
|
||||
DataTypes element_types;
|
||||
element_types.reserve(x.size());
|
||||
|
||||
bool has_signed_int = false;
|
||||
bool uint64_convert_possible = true;
|
||||
for (const Field & elem : x)
|
||||
{
|
||||
DataTypePtr type = applyVisitor(*this, elem);
|
||||
element_types.emplace_back(type);
|
||||
checkUInt64ToIn64Conversion(has_signed_int, uint64_convert_possible, type, elem);
|
||||
}
|
||||
|
||||
if (has_signed_int && uint64_convert_possible)
|
||||
convertUInt64ToInt64IfPossible(element_types);
|
||||
element_types.emplace_back(applyVisitor(*this, elem));
|
||||
|
||||
return std::make_shared<DataTypeArray>(getLeastSupertype<on_error>(element_types));
|
||||
}
|
||||
@ -174,28 +166,14 @@ DataTypePtr FieldToDataType<on_error>::operator() (const Map & map) const
|
||||
key_types.reserve(map.size());
|
||||
value_types.reserve(map.size());
|
||||
|
||||
bool k_has_signed_int = false;
|
||||
bool k_uint64_convert_possible = true;
|
||||
bool v_has_signed_int = false;
|
||||
bool v_uint64_convert_possible = true;
|
||||
for (const auto & elem : map)
|
||||
{
|
||||
const auto & tuple = elem.safeGet<const Tuple &>();
|
||||
assert(tuple.size() == 2);
|
||||
DataTypePtr k_type = applyVisitor(*this, tuple[0]);
|
||||
key_types.push_back(k_type);
|
||||
checkUInt64ToIn64Conversion(k_has_signed_int, k_uint64_convert_possible, k_type, tuple[0]);
|
||||
DataTypePtr v_type = applyVisitor(*this, tuple[1]);
|
||||
value_types.push_back(v_type);
|
||||
checkUInt64ToIn64Conversion(v_has_signed_int, v_uint64_convert_possible, v_type, tuple[1]);
|
||||
key_types.push_back(applyVisitor(*this, tuple[0]));
|
||||
value_types.push_back(applyVisitor(*this, tuple[1]));
|
||||
}
|
||||
|
||||
if (k_has_signed_int && k_uint64_convert_possible)
|
||||
convertUInt64ToInt64IfPossible(key_types);
|
||||
|
||||
if (v_has_signed_int && v_uint64_convert_possible)
|
||||
convertUInt64ToInt64IfPossible(value_types);
|
||||
|
||||
return std::make_shared<DataTypeMap>(
|
||||
getLeastSupertype<on_error>(key_types),
|
||||
getLeastSupertype<on_error>(value_types));
|
||||
@ -227,28 +205,6 @@ DataTypePtr FieldToDataType<on_error>::operator()(const bool &) const
|
||||
return DataTypeFactory::instance().get("Bool");
|
||||
}
|
||||
|
||||
template <LeastSupertypeOnError on_error>
|
||||
void FieldToDataType<on_error>::checkUInt64ToIn64Conversion(bool & has_signed_int, bool & uint64_convert_possible, const DataTypePtr & type, const Field & elem) const
|
||||
{
|
||||
if (uint64_convert_possible)
|
||||
{
|
||||
bool is_native_int = WhichDataType(type).isNativeInt();
|
||||
|
||||
if (is_native_int)
|
||||
has_signed_int |= is_native_int;
|
||||
else if (type->getTypeId() == TypeIndex::UInt64)
|
||||
uint64_convert_possible &= (elem.template get<UInt64>() <= std::numeric_limits<Int64>::max());
|
||||
}
|
||||
}
|
||||
|
||||
template <LeastSupertypeOnError on_error>
|
||||
void FieldToDataType<on_error>::convertUInt64ToInt64IfPossible(DataTypes & data_types) const
|
||||
{
|
||||
for (auto& type : data_types)
|
||||
if (type->getTypeId() == TypeIndex::UInt64)
|
||||
type = std::make_shared<DataTypeInt64>();
|
||||
}
|
||||
|
||||
template class FieldToDataType<LeastSupertypeOnError::Throw>;
|
||||
template class FieldToDataType<LeastSupertypeOnError::String>;
|
||||
template class FieldToDataType<LeastSupertypeOnError::Null>;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user