mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge branch 'tostring-fix' of https://github.com/bigo-sg/ClickHouse into tostring-fix
This commit is contained in:
zhanglistar
commit
46bcfd4fca
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
@ -12,7 +12,7 @@ tests/ci/cancel_and_rerun_workflow_lambda/app.py
|
||||
- Backward Incompatible Change
|
||||
- Build/Testing/Packaging Improvement
|
||||
- Documentation (changelog entry is not required)
|
||||
- Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
|
||||
- Critical Bug Fix (crash, data loss, RBAC)
|
||||
- Bug Fix (user-visible misbehavior in an official stable release)
|
||||
- CI Fix or Improvement (changelog entry is not required)
|
||||
- Not for changelog (changelog entry is not required)
|
||||
|
||||
7
.github/workflows/merge_queue.yml
vendored
7
.github/workflows/merge_queue.yml
vendored
@ -58,13 +58,8 @@ jobs:
|
||||
test_name: Style check
|
||||
runner_type: style-checker-aarch64
|
||||
run_command: |
|
||||
python3 style_check.py
|
||||
python3 style_check.py --no-push
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
secrets:
|
||||
secret_envs: |
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
|
||||
RCSK
|
||||
FastTest:
|
||||
needs: [RunConfig, BuildDockers]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
|
||||
|
||||
34
.github/workflows/nightly.yml
vendored
34
.github/workflows/nightly.yml
vendored
@ -27,7 +27,7 @@ jobs:
|
||||
id: runconfig
|
||||
run: |
|
||||
echo "::group::configure CI run"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --skip-jobs --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --workflow NightlyBuilds --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::CI run configure results"
|
||||
@ -44,9 +44,39 @@ jobs:
|
||||
with:
|
||||
data: "${{ needs.RunConfig.outputs.data }}"
|
||||
set_latest: true
|
||||
|
||||
Builds_1:
|
||||
needs: [RunConfig]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
|
||||
uses: ./.github/workflows/reusable_build_stage.yml
|
||||
with:
|
||||
stage: Builds_1
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
Tests_1:
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
stage: Tests_1
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
Builds_2:
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }}
|
||||
uses: ./.github/workflows/reusable_build_stage.yml
|
||||
with:
|
||||
stage: Builds_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
Tests_2:
|
||||
needs: [RunConfig, Builds_1, Tests_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
stage: Tests_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
CheckWorkflow:
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [RunConfig, BuildDockers]
|
||||
needs: [RunConfig, BuildDockers, Tests_2]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
|
||||
5
.github/workflows/pull_request.yml
vendored
5
.github/workflows/pull_request.yml
vendored
@ -79,10 +79,7 @@ jobs:
|
||||
python3 style_check.py
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
secrets:
|
||||
secret_envs: |
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
|
||||
RCSK
|
||||
robot_git_token: ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
|
||||
FastTest:
|
||||
needs: [RunConfig, BuildDockers, StyleCheck]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
|
||||
|
||||
17
.github/workflows/reusable_build.yml
vendored
17
.github/workflows/reusable_build.yml
vendored
@ -34,8 +34,11 @@ name: Build ClickHouse
|
||||
description: additional ENV variables to setup the job
|
||||
type: string
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
@ -58,10 +61,18 @@ jobs:
|
||||
run: |
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
${{inputs.additional_envs}}
|
||||
${{secrets.secret_envs}}
|
||||
DOCKER_TAG<<DOCKER_JSON
|
||||
${{ toJson(fromJson(inputs.data).docker_data.images) }}
|
||||
DOCKER_JSON
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.robot_git_token}}
|
||||
RCSK
|
||||
CI_DB_URL<<CIDBU
|
||||
${{ secrets.ci_db_url }}
|
||||
CIDBU
|
||||
CI_DB_PASSWORD<<CIDBP
|
||||
${{ secrets.ci_db_password }}
|
||||
CIDBP
|
||||
EOF
|
||||
python3 "$GITHUB_WORKSPACE"/tests/ci/ci_config.py --build-name "${{inputs.build_name}}" >> "$GITHUB_ENV"
|
||||
- name: Apply sparse checkout for contrib # in order to check that it doesn't break build
|
||||
|
||||
11
.github/workflows/reusable_build_stage.yml
vendored
11
.github/workflows/reusable_build_stage.yml
vendored
@ -18,8 +18,11 @@ name: BuildStageWF
|
||||
type: string
|
||||
required: true
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
@ -39,4 +42,6 @@ jobs:
|
||||
checkout_depth: 0
|
||||
data: ${{ inputs.data }}
|
||||
secrets:
|
||||
secret_envs: ${{ secrets.secret_envs }}
|
||||
robot_git_token: ${{ secrets.robot_git_token }}
|
||||
ci_db_url: ${{ secrets.ci_db_url }}
|
||||
ci_db_password: ${{ secrets.ci_db_password }}
|
||||
|
||||
17
.github/workflows/reusable_simple_job.yml
vendored
17
.github/workflows/reusable_simple_job.yml
vendored
@ -45,8 +45,11 @@ name: Simple job
|
||||
type: boolean
|
||||
default: false
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
|
||||
@ -77,7 +80,15 @@ jobs:
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
CHECK_NAME=${{ inputs.test_name }}
|
||||
${{inputs.additional_envs}}
|
||||
${{secrets.secret_envs}}
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.robot_git_token}}
|
||||
RCSK
|
||||
CI_DB_URL<<CIDBU
|
||||
${{ secrets.ci_db_url }}
|
||||
CIDBU
|
||||
CI_DB_PASSWORD<<CIDBP
|
||||
${{ secrets.ci_db_password }}
|
||||
CIDBP
|
||||
EOF
|
||||
- name: Common setup
|
||||
uses: ./.github/actions/common_setup
|
||||
|
||||
17
.github/workflows/reusable_test.yml
vendored
17
.github/workflows/reusable_test.yml
vendored
@ -40,8 +40,11 @@ name: Testing workflow
|
||||
type: string
|
||||
default: "$GITHUB_WORKSPACE/tests/ci"
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
|
||||
@ -75,10 +78,18 @@ jobs:
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
CHECK_NAME=${{ inputs.test_name }}
|
||||
${{inputs.additional_envs}}
|
||||
${{secrets.secret_envs}}
|
||||
DOCKER_TAG<<DOCKER_JSON
|
||||
${{ toJson(fromJson(inputs.data).docker_data.images) }}
|
||||
DOCKER_JSON
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.robot_git_token}}
|
||||
RCSK
|
||||
CI_DB_URL<<CIDBU
|
||||
${{ secrets.ci_db_url }}
|
||||
CIDBU
|
||||
CI_DB_PASSWORD<<CIDBP
|
||||
${{ secrets.ci_db_password }}
|
||||
CIDBP
|
||||
EOF
|
||||
- name: Common setup
|
||||
uses: ./.github/actions/common_setup
|
||||
|
||||
11
.github/workflows/reusable_test_stage.yml
vendored
11
.github/workflows/reusable_test_stage.yml
vendored
@ -15,8 +15,11 @@ name: StageWF
|
||||
type: string
|
||||
required: true
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
@ -32,4 +35,6 @@ jobs:
|
||||
runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
|
||||
data: ${{ inputs.data }}
|
||||
secrets:
|
||||
secret_envs: ${{ secrets.secret_envs }}
|
||||
robot_git_token: ${{ secrets.robot_git_token }}
|
||||
ci_db_url: ${{ secrets.ci_db_url }}
|
||||
ci_db_password: ${{ secrets.ci_db_password }}
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -159,6 +159,7 @@ website/package-lock.json
|
||||
/programs/server/store
|
||||
/programs/server/uuid
|
||||
/programs/server/coordination
|
||||
/programs/server/workload
|
||||
|
||||
# temporary test files
|
||||
tests/queries/0_stateless/test_*
|
||||
|
||||
5
.gitmodules
vendored
5
.gitmodules
vendored
@ -1,6 +1,9 @@
|
||||
# Please do not use 'branch = ...' tags with submodule entries. Such tags make updating submodules a
|
||||
# little bit more convenient but they do *not* specify the tracked submodule branch. Thus, they are
|
||||
# more confusing than useful.
|
||||
[submodule "contrib/jwt-cpp"]
|
||||
path = contrib/jwt-cpp
|
||||
url = https://github.com/Thalhammer/jwt-cpp
|
||||
[submodule "contrib/zstd"]
|
||||
path = contrib/zstd
|
||||
url = https://github.com/facebook/zstd
|
||||
@ -332,7 +335,7 @@
|
||||
url = https://github.com/ClickHouse/usearch.git
|
||||
[submodule "contrib/SimSIMD"]
|
||||
path = contrib/SimSIMD
|
||||
url = https://github.com/ashvardanian/SimSIMD.git
|
||||
url = https://github.com/ClickHouse/SimSIMD.git
|
||||
[submodule "contrib/FP16"]
|
||||
path = contrib/FP16
|
||||
url = https://github.com/Maratyszcza/FP16.git
|
||||
|
||||
162
CHANGELOG.md
162
CHANGELOG.md
@ -1,4 +1,5 @@
|
||||
### Table of Contents
|
||||
**[ClickHouse release v24.10, 2024-10-31](#2410)**<br/>
|
||||
**[ClickHouse release v24.9, 2024-09-26](#249)**<br/>
|
||||
**[ClickHouse release v24.8 LTS, 2024-08-20](#248)**<br/>
|
||||
**[ClickHouse release v24.7, 2024-07-30](#247)**<br/>
|
||||
@ -12,6 +13,165 @@
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### <a id="2410"></a> ClickHouse release 24.10, 2024-10-31
|
||||
|
||||
#### Backward Incompatible Change
|
||||
* Allow to write `SETTINGS` before `FORMAT` in a chain of queries with `UNION` when subqueries are inside parentheses. This closes [#39712](https://github.com/ClickHouse/ClickHouse/issues/39712). Change the behavior when a query has the SETTINGS clause specified twice in a sequence. The closest SETTINGS clause will have a preference for the corresponding subquery. In the previous versions, the outermost SETTINGS clause could take a preference over the inner one. [#68614](https://github.com/ClickHouse/ClickHouse/pull/68614) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Reordering of filter conditions from `[PRE]WHERE` clause is now allowed by default. It could be disabled by setting `allow_reorder_prewhere_conditions` to `false`. [#70657](https://github.com/ClickHouse/ClickHouse/pull/70657) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Remove the `idxd-config` library, which has an incompatible license. This also removes the experimental Intel DeflateQPL codec. [#70987](https://github.com/ClickHouse/ClickHouse/pull/70987) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### New Feature
|
||||
* Allow to grant access to the wildcard prefixes. `GRANT SELECT ON db.table_pefix_* TO user`. [#65311](https://github.com/ClickHouse/ClickHouse/pull/65311) ([pufit](https://github.com/pufit)).
|
||||
* If you press space bar during query runtime, the client will display a real-time table with detailed metrics. You can enable it globally with the new `--progress-table` option in clickhouse-client; a new `--enable-progress-table-toggle` is associated with the `--progress-table` option, and toggles the rendering of the progress table by pressing the control key (Space). [#63689](https://github.com/ClickHouse/ClickHouse/pull/63689) ([Maria Khristenko](https://github.com/mariaKhr)), [#70423](https://github.com/ClickHouse/ClickHouse/pull/70423) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Allow to cache read files for object storage table engines and data lakes using hash from ETag + file path as cache key. [#70135](https://github.com/ClickHouse/ClickHouse/pull/70135) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Support creating a table with a query: `CREATE TABLE ... CLONE AS ...`. It clones the source table's schema and then attaches all partitions to the newly created table. This feature is only supported with tables of the `MergeTree` family Closes [#65015](https://github.com/ClickHouse/ClickHouse/issues/65015). [#69091](https://github.com/ClickHouse/ClickHouse/pull/69091) ([tuanpach](https://github.com/tuanpach)).
|
||||
* Add a new system table, `system.query_metric_log` which contains history of memory and metric values from table system.events for individual queries, periodically flushed to disk. [#66532](https://github.com/ClickHouse/ClickHouse/pull/66532) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* A simple SELECT query can be written with implicit SELECT to enable calculator-style expressions, e.g., `ch "1 + 2"`. This is controlled by a new setting, `implicit_select`. [#68502](https://github.com/ClickHouse/ClickHouse/pull/68502) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Support the `--copy` mode for clickhouse local as a shortcut for format conversion [#68503](https://github.com/ClickHouse/ClickHouse/issues/68503). [#68583](https://github.com/ClickHouse/ClickHouse/pull/68583) ([Denis Hananein](https://github.com/denis-hananein)).
|
||||
* Add a builtin HTML page for visualizing merges which is available at the `/merges` path. [#70821](https://github.com/ClickHouse/ClickHouse/pull/70821) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add support for `arrayUnion` function. [#68989](https://github.com/ClickHouse/ClickHouse/pull/68989) ([Peter Nguyen](https://github.com/petern48)).
|
||||
* Allow parametrised SQL aliases. [#50665](https://github.com/ClickHouse/ClickHouse/pull/50665) ([Anton Kozlov](https://github.com/tonickkozlov)).
|
||||
* A new aggregate function `quantileExactWeightedInterpolated`, which is a interpolated version based on quantileExactWeighted. Some people may wonder why we need a new `quantileExactWeightedInterpolated` since we already have `quantileExactInterpolatedWeighted`. The reason is the new one is more accurate than the old one. This is for spark compatibility. [#69619](https://github.com/ClickHouse/ClickHouse/pull/69619) ([李扬](https://github.com/taiyang-li)).
|
||||
* A new function `arrayElementOrNull`. It returns `NULL` if the array index is out of range or a Map key not found. [#69646](https://github.com/ClickHouse/ClickHouse/pull/69646) ([李扬](https://github.com/taiyang-li)).
|
||||
* Allows users to specify regular expressions through new `message_regexp` and `message_regexp_negative` fields in the `config.xml` file to filter out logging. The logging is applied to the formatted un-colored text for the most intuitive developer experience. [#69657](https://github.com/ClickHouse/ClickHouse/pull/69657) ([Peter Nguyen](https://github.com/petern48)).
|
||||
* Added `RIPEMD160` function, which computes the RIPEMD-160 cryptographic hash of a string. Example: `SELECT HEX(RIPEMD160('The quick brown fox jumps over the lazy dog'))` returns `37F332F68DB77BD9D7EDD4969571AD671CF9DD3B`. [#70087](https://github.com/ClickHouse/ClickHouse/pull/70087) ([Dergousov Maxim](https://github.com/m7kss1)).
|
||||
* Support reading `Iceberg` tables on `HDFS`. [#70268](https://github.com/ClickHouse/ClickHouse/pull/70268) ([flynn](https://github.com/ucasfl)).
|
||||
* Support for CTE in the form of `WITH ... INSERT`, as previously we only supported `INSERT ... WITH ...`. [#70593](https://github.com/ClickHouse/ClickHouse/pull/70593) ([Shichao Jin](https://github.com/jsc0218)).
|
||||
* MongoDB integration: support for all MongoDB types, support for WHERE and ORDER BY statements on MongoDB side, restriction for expressions unsupported by MongoDB. Note that the new inegration is disabled by default, to use it, please set `<use_legacy_mongodb_integration>` to `false` in server config. [#63279](https://github.com/ClickHouse/ClickHouse/pull/63279) ([Kirill Nikiforov](https://github.com/allmazz)).
|
||||
* A new function `getSettingOrDefault` added to return the default value and avoid exception if a custom setting is not found in the current profile. [#69917](https://github.com/ClickHouse/ClickHouse/pull/69917) ([Shankar](https://github.com/shiyer7474)).
|
||||
|
||||
#### Experimental feature
|
||||
* Refreshable materialized views are production ready. [#70550](https://github.com/ClickHouse/ClickHouse/pull/70550) ([Michael Kolupaev](https://github.com/al13n321)). Refreshable materialized views are now supported in Replicated databases. [#60669](https://github.com/ClickHouse/ClickHouse/pull/60669) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Parallel replicas are moved from experimental to beta. Reworked settings that control the behavior of parallel replicas algorithms. A quick recap: ClickHouse has four different algorithms for parallel reading involving multiple replicas, which is reflected in the setting `parallel_replicas_mode`, the default value for it is `read_tasks` Additionally, the toggle-switch setting `enable_parallel_replicas` has been added. [#63151](https://github.com/ClickHouse/ClickHouse/pull/63151) ([Alexey Milovidov](https://github.com/alexey-milovidov)), ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Support for the `Dynamic` type in most functions by executing them on internal types inside `Dynamic`. [#69691](https://github.com/ClickHouse/ClickHouse/pull/69691) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Allow to read/write the `JSON` type as a binary string in `RowBinary` format under settings `input_format_binary_read_json_as_string/output_format_binary_write_json_as_string`. [#70288](https://github.com/ClickHouse/ClickHouse/pull/70288) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Allow to serialize/deserialize `JSON` column as single String column in the Native format. For output use setting `output_format_native_write_json_as_string`. For input, use serialization version `1` before the column data. [#70312](https://github.com/ClickHouse/ClickHouse/pull/70312) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Introduced a special (experimental) mode of a merge selector for MergeTree tables which makes it more aggressive for the partitions that are close to the limit by the number of parts. It is controlled by the `merge_selector_use_blurry_base` MergeTree-level setting. [#70645](https://github.com/ClickHouse/ClickHouse/pull/70645) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Implement generic ser/de between Avro's `Union` and ClickHouse's `Variant` types. Resolves [#69713](https://github.com/ClickHouse/ClickHouse/issues/69713). [#69712](https://github.com/ClickHouse/ClickHouse/pull/69712) ([Jiří Kozlovský](https://github.com/jirislav)).
|
||||
|
||||
#### Performance Improvement
|
||||
* Refactor `IDisk` and `IObjectStorage` for better performance. Tables from `plain` and `plain_rewritable` object storages will initialize faster. [#68146](https://github.com/ClickHouse/ClickHouse/pull/68146) ([Alexey Milovidov](https://github.com/alexey-milovidov), [Julia Kartseva](https://github.com/jkartseva)). Do not call the LIST object storage API when determining if a file or directory exists on the plain rewritable disk, as it can be cost-inefficient. [#70852](https://github.com/ClickHouse/ClickHouse/pull/70852) ([Julia Kartseva](https://github.com/jkartseva)). Reduce the number of object storage HEAD API requests in the plain_rewritable disk. [#70915](https://github.com/ClickHouse/ClickHouse/pull/70915) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Added an ability to parse data directly into sparse columns. [#69828](https://github.com/ClickHouse/ClickHouse/pull/69828) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Improved performance of parsing formats with high number of missed values (e.g. `JSONEachRow`). [#69875](https://github.com/ClickHouse/ClickHouse/pull/69875) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Supports parallel reading of parquet row groups and prefetching of row groups in single-threaded mode. [#69862](https://github.com/ClickHouse/ClickHouse/pull/69862) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Support minmax index for `pointInPolygon`. [#62085](https://github.com/ClickHouse/ClickHouse/pull/62085) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Use bloom filters when reading Parquet files. [#62966](https://github.com/ClickHouse/ClickHouse/pull/62966) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* Lock-free parts rename to avoid INSERT affect SELECT (due to parts lock) (under normal circumstances with `fsync_part_directory`, QPS of SELECT with INSERT in parallel, increased 2x, under heavy load the effect is even bigger). Note, this only includes `ReplicatedMergeTree` for now. [#64955](https://github.com/ClickHouse/ClickHouse/pull/64955) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Respect `ttl_only_drop_parts` on `materialize ttl`; only read necessary columns to recalculate TTL and drop parts by replacing them with an empty one. [#65488](https://github.com/ClickHouse/ClickHouse/pull/65488) ([Andrey Zvonov](https://github.com/zvonand)).
|
||||
* Optimized thread creation in the ThreadPool to minimize lock contention. Thread creation is now performed outside of the critical section to avoid delays in job scheduling and thread management under high load conditions. This leads to a much more responsive ClickHouse under heavy concurrent load. [#68694](https://github.com/ClickHouse/ClickHouse/pull/68694) ([filimonov](https://github.com/filimonov)).
|
||||
* Enable reading `LowCardinality` string columns from `ORC`. [#69481](https://github.com/ClickHouse/ClickHouse/pull/69481) ([李扬](https://github.com/taiyang-li)).
|
||||
* Use `LowCardinality` for `ProfileEvents` in system logs such as `part_log`, `query_views_log`, `filesystem_cache_log`. [#70152](https://github.com/ClickHouse/ClickHouse/pull/70152) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Improve performance of `fromUnixTimestamp`/`toUnixTimestamp` functions. [#71042](https://github.com/ClickHouse/ClickHouse/pull/71042) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Don't disable nonblocking read from page cache for the entire server when reading from a blocking I/O. This was leading to a poorer performance when a single filesystem (e.g., tmpfs) didn't support the `preadv2` syscall while others do. [#70299](https://github.com/ClickHouse/ClickHouse/pull/70299) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* `ALTER TABLE .. REPLACE PARTITION` doesn't wait anymore for mutations/merges that happen in other partitions. [#59138](https://github.com/ClickHouse/ClickHouse/pull/59138) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Don't do validation when synchronizing ACL from Keeper. It's validating during creation. It shouldn't matter that much, but there are installations with tens of thousands or even more user created, and the unnecessary hash validation can take a long time to finish during server startup (it synchronizes everything from keeper). [#70644](https://github.com/ClickHouse/ClickHouse/pull/70644) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### Improvement
|
||||
* `CREATE TABLE AS` will copy `PRIMARY KEY`, `ORDER BY`, and similar clauses (of `MergeTree` tables). [#69739](https://github.com/ClickHouse/ClickHouse/pull/69739) ([sakulali](https://github.com/sakulali)).
|
||||
* Support 64-bit XID in Keeper. It can be enabled with the `use_xid_64` configuration value. [#69908](https://github.com/ClickHouse/ClickHouse/pull/69908) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Command-line arguments for Bool settings are set to true when no value is provided for the argument (e.g. `clickhouse-client --optimize_aggregation_in_order --query "SELECT 1"`). [#70459](https://github.com/ClickHouse/ClickHouse/pull/70459) ([davidtsuk](https://github.com/davidtsuk)).
|
||||
* Added user-level settings `min_free_disk_bytes_to_throw_insert` and `min_free_disk_ratio_to_throw_insert` to prevent insertions on disks that are almost full. [#69755](https://github.com/ClickHouse/ClickHouse/pull/69755) ([Marco Vilas Boas](https://github.com/marco-vb)).
|
||||
* Embedded documentation for settings will be strictly more detailed and complete than the documentation on the website. This is the first step before making the website documentation always auto-generated from the source code. This has long-standing implications: - it will be guaranteed to have every setting; - there is no chance of having default values obsolete; - we can generate this documentation for each ClickHouse version; - the documentation can be displayed by the server itself even without Internet access. Generate the docs on the website from the source code. [#70289](https://github.com/ClickHouse/ClickHouse/pull/70289) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Allow empty needle in the function `replace`, the same behavior with PostgreSQL. [#69918](https://github.com/ClickHouse/ClickHouse/pull/69918) ([zhanglistar](https://github.com/zhanglistar)).
|
||||
* Allow empty needle in functions `replaceRegexp*`. [#70053](https://github.com/ClickHouse/ClickHouse/pull/70053) ([zhanglistar](https://github.com/zhanglistar)).
|
||||
* Symbolic links for tables in the `data/database_name/` directory are created for the actual paths to the table's data, depending on the storage policy, instead of the `store/...` directory on the default disk. [#61777](https://github.com/ClickHouse/ClickHouse/pull/61777) ([Kirill](https://github.com/kirillgarbar)).
|
||||
* While parsing an `Enum` field from `JSON`, a string containing an integer will be interpreted as the corresponding `Enum` element. This closes [#65119](https://github.com/ClickHouse/ClickHouse/issues/65119). [#66801](https://github.com/ClickHouse/ClickHouse/pull/66801) ([scanhex12](https://github.com/scanhex12)).
|
||||
* Allow `TRIM` -ing `LEADING` or `TRAILING` empty string as a no-op. Closes [#67792](https://github.com/ClickHouse/ClickHouse/issues/67792). [#68455](https://github.com/ClickHouse/ClickHouse/pull/68455) ([Peter Nguyen](https://github.com/petern48)).
|
||||
* Improve compatibility of `cast(timestamp as String)` with Spark. [#69179](https://github.com/ClickHouse/ClickHouse/pull/69179) ([Wenzheng Liu](https://github.com/lwz9103)).
|
||||
* Always use the new analyzer to calculate constant expressions when `enable_analyzer` is set to `true`. Support calculation of `executable` table function arguments without using `SELECT` query for constant expressions. [#69292](https://github.com/ClickHouse/ClickHouse/pull/69292) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Add a setting `enable_secure_identifiers` to disallow identifiers with special characters. [#69411](https://github.com/ClickHouse/ClickHouse/pull/69411) ([tuanpach](https://github.com/tuanpach)).
|
||||
* Add `show_create_query_identifier_quoting_rule` to define identifier quoting behavior in the `SHOW CREATE TABLE` query result. Possible values: - `user_display`: When the identifiers is a keyword. - `when_necessary`: When the identifiers is one of `{"distinct", "all", "table"}` and when it could lead to ambiguity: column names, dictionary attribute names. - `always`: Always quote identifiers. [#69448](https://github.com/ClickHouse/ClickHouse/pull/69448) ([tuanpach](https://github.com/tuanpach)).
|
||||
* Improve restoring of access entities' dependencies [#69563](https://github.com/ClickHouse/ClickHouse/pull/69563) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* If you run `clickhouse-client` or other CLI application, and it starts up slowly due to an overloaded server, and you start typing your query, such as `SELECT`, the previous versions will display the remaining of the terminal echo contents before printing the greetings message, such as `SELECTClickHouse local version 24.10.1.1.` instead of `ClickHouse local version 24.10.1.1.`. Now it is fixed. This closes [#31696](https://github.com/ClickHouse/ClickHouse/issues/31696). [#69856](https://github.com/ClickHouse/ClickHouse/pull/69856) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add new column `readonly_duration` to the `system.replicas` table. Needed to be able to distinguish actual readonly replicas from sentinel ones in alerts. [#69871](https://github.com/ClickHouse/ClickHouse/pull/69871) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Change the type of `join_output_by_rowlist_perkey_rows_threshold` setting type to unsigned integer. [#69886](https://github.com/ClickHouse/ClickHouse/pull/69886) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Enhance OpenTelemetry span logging to include query settings. [#70011](https://github.com/ClickHouse/ClickHouse/pull/70011) ([sharathks118](https://github.com/sharathks118)).
|
||||
* Add diagnostic info about higher-order array functions if lambda result type is unexpected. [#70093](https://github.com/ClickHouse/ClickHouse/pull/70093) ([ttanay](https://github.com/ttanay)).
|
||||
* Keeper improvement: less locking during cluster changes. [#70275](https://github.com/ClickHouse/ClickHouse/pull/70275) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Add `WITH IMPLICIT` and `FINAL` keywords to the `SHOW GRANTS` command. Fix a minor bug with implicit grants: [#70094](https://github.com/ClickHouse/ClickHouse/issues/70094). [#70293](https://github.com/ClickHouse/ClickHouse/pull/70293) ([pufit](https://github.com/pufit)).
|
||||
* Respect `compatibility` for MergeTree settings. The `compatibility` value is taken from the `default` profile on server startup, and default MergeTree settings are changed accordingly. Further changes of the `compatibility` setting do not affect MergeTree settings. [#70322](https://github.com/ClickHouse/ClickHouse/pull/70322) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Avoid spamming the logs with large HTTP response bodies in case of errors during inter-server communication. [#70487](https://github.com/ClickHouse/ClickHouse/pull/70487) ([Vladimir Cherkasov](https://github.com/vdimir)).
|
||||
* Added a new setting `max_parts_to_move` to control the maximum number of parts that can be moved at once. [#70520](https://github.com/ClickHouse/ClickHouse/pull/70520) ([Vladimir Cherkasov](https://github.com/vdimir)).
|
||||
* Limit the frequency of certain log messages. [#70601](https://github.com/ClickHouse/ClickHouse/pull/70601) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* `CHECK TABLE` with `PART` qualifier was incorrectly formatted in the client. [#70660](https://github.com/ClickHouse/ClickHouse/pull/70660) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Support writing the column index and the offset index using parquet native writer. [#70669](https://github.com/ClickHouse/ClickHouse/pull/70669) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Support parsing `DateTime64` for microsecond and timezone in joda syntax ("joda" is a popular Java library for date and time, and the "joda syntax" is that library's style). [#70737](https://github.com/ClickHouse/ClickHouse/pull/70737) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Changed an approach to figure out if a cloud storage supports [batch delete](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) or not. [#70786](https://github.com/ClickHouse/ClickHouse/pull/70786) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Support for Parquet page v2 in the native reader. [#70807](https://github.com/ClickHouse/ClickHouse/pull/70807) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* A check if table has both `storage_policy` and `disk` set. A check if a new storage policy is compatible with an old one when using `disk` setting is added. [#70839](https://github.com/ClickHouse/ClickHouse/pull/70839) ([Kirill](https://github.com/kirillgarbar)).
|
||||
* Add `system.s3_queue_settings` and `system.azure_queue_settings`. [#70841](https://github.com/ClickHouse/ClickHouse/pull/70841) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Functions `base58Encode` and `base58Decode` now accept arguments of type `FixedString`. Example: `SELECT base58Encode(toFixedString('plaintext', 9));`. [#70846](https://github.com/ClickHouse/ClickHouse/pull/70846) ([Faizan Patel](https://github.com/faizan2786)).
|
||||
* Add the `partition` column to every entry type of the part log. Previously, it was set only for some entries. This closes [#70819](https://github.com/ClickHouse/ClickHouse/issues/70819). [#70848](https://github.com/ClickHouse/ClickHouse/pull/70848) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add `MergeStart` and `MutateStart` events into `system.part_log` which helps with merges analysis and visualization. [#70850](https://github.com/ClickHouse/ClickHouse/pull/70850) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add a profile event about the number of merged source parts. It allows the monitoring of the fanout of the merge tree in production. [#70908](https://github.com/ClickHouse/ClickHouse/pull/70908) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Background downloads to the filesystem cache were enabled back. [#70929](https://github.com/ClickHouse/ClickHouse/pull/70929) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Add a new merge selector algorithm, named `Trivial`, for professional usage only. It is worse than the `Simple` merge selector. [#70969](https://github.com/ClickHouse/ClickHouse/pull/70969) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Support for atomic `CREATE OR REPLACE VIEW`. [#70536](https://github.com/ClickHouse/ClickHouse/pull/70536) ([tuanpach](https://github.com/tuanpach))
|
||||
* Added `strict_once` mode to aggregate function `windowFunnel` to avoid counting one event several times in case it matches multiple conditions, close [#21835](https://github.com/ClickHouse/ClickHouse/issues/21835). [#69738](https://github.com/ClickHouse/ClickHouse/pull/69738) ([Vladimir Cherkasov](https://github.com/vdimir)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Apply configuration updates in global context object. It fixes issues like [#62308](https://github.com/ClickHouse/ClickHouse/issues/62308). [#62944](https://github.com/ClickHouse/ClickHouse/pull/62944) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix `ReadSettings` not using user set values, because defaults were only used. [#65625](https://github.com/ClickHouse/ClickHouse/pull/65625) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix type mismatch issue in `sumMapFiltered` when using signed arguments. [#58408](https://github.com/ClickHouse/ClickHouse/pull/58408) ([Chen768959](https://github.com/Chen768959)).
|
||||
* Fix toHour-like conversion functions' monotonicity when optional time zone argument is passed. [#60264](https://github.com/ClickHouse/ClickHouse/pull/60264) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Relax `supportsPrewhere` check for `Merge` tables. This fixes [#61064](https://github.com/ClickHouse/ClickHouse/issues/61064). It was hardened unnecessarily in [#60082](https://github.com/ClickHouse/ClickHouse/issues/60082). [#61091](https://github.com/ClickHouse/ClickHouse/pull/61091) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix `use_concurrency_control` setting handling for proper `concurrent_threads_soft_limit_num` limit enforcing. This enables concurrency control by default because previously it was broken. [#61473](https://github.com/ClickHouse/ClickHouse/pull/61473) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Fix incorrect `JOIN ON` section optimization in case of `IS NULL` check under any other function (like `NOT`) that may lead to wrong results. Closes [#67915](https://github.com/ClickHouse/ClickHouse/issues/67915). [#68049](https://github.com/ClickHouse/ClickHouse/pull/68049) ([Vladimir Cherkasov](https://github.com/vdimir)).
|
||||
* Prevent `ALTER` queries that would make the `CREATE` query of tables invalid. [#68574](https://github.com/ClickHouse/ClickHouse/pull/68574) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Fix inconsistent AST formatting for `negate` (`-`) and `NOT` functions with tuples and arrays. [#68600](https://github.com/ClickHouse/ClickHouse/pull/68600) ([Vladimir Cherkasov](https://github.com/vdimir)).
|
||||
* Fix insertion of incomplete type into `Dynamic` during deserialization. It could lead to `Parameter out of bound` errors. [#69291](https://github.com/ClickHouse/ClickHouse/pull/69291) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Zero-copy replication, which is experimental and should not be used in production: fix inf loop after `restore replica` in the replicated merge tree with zero copy. [#69293](https://github.com/CljmnickHouse/ClickHouse/pull/69293) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Return back default value of `processing_threads_num` as number of cpu cores in storage `S3Queue`. [#69384](https://github.com/ClickHouse/ClickHouse/pull/69384) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Bypass try/catch flow when de/serializing nested repeated protobuf to nested columns (fixes [#41971](https://github.com/ClickHouse/ClickHouse/issues/41971)). [#69556](https://github.com/ClickHouse/ClickHouse/pull/69556) ([Eliot Hautefeuille](https://github.com/hileef)).
|
||||
* Fix crash during insertion into FixedString column in PostgreSQL engine. [#69584](https://github.com/ClickHouse/ClickHouse/pull/69584) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix crash when executing `create view t as (with recursive 42 as ttt select ttt);`. [#69676](https://github.com/ClickHouse/ClickHouse/pull/69676) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Fixed `maxMapState` throwing 'Bad get' if value type is DateTime64. [#69787](https://github.com/ClickHouse/ClickHouse/pull/69787) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix `getSubcolumn` with `LowCardinality` columns by overriding `useDefaultImplementationForLowCardinalityColumns` to return `true`. [#69831](https://github.com/ClickHouse/ClickHouse/pull/69831) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Fix permanent blocked distributed sends if a DROP of distributed table failed. [#69843](https://github.com/ClickHouse/ClickHouse/pull/69843) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix non-cancellable queries containing WITH FILL with NaN keys. This closes [#69261](https://github.com/ClickHouse/ClickHouse/issues/69261). [#69845](https://github.com/ClickHouse/ClickHouse/pull/69845) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix analyzer default with old compatibility value. [#69895](https://github.com/ClickHouse/ClickHouse/pull/69895) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Don't check dependencies during CREATE OR REPLACE VIEW during DROP of old table. Previously CREATE OR REPLACE query failed when there are dependent tables of the recreated view. [#69907](https://github.com/ClickHouse/ClickHouse/pull/69907) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Something for Decimal. Fixes [#69730](https://github.com/ClickHouse/ClickHouse/issues/69730). [#69978](https://github.com/ClickHouse/ClickHouse/pull/69978) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* Now DEFINER/INVOKER will work with parameterized views. [#69984](https://github.com/ClickHouse/ClickHouse/pull/69984) ([pufit](https://github.com/pufit)).
|
||||
* Fix parsing for view's definers. [#69985](https://github.com/ClickHouse/ClickHouse/pull/69985) ([pufit](https://github.com/pufit)).
|
||||
* Fixed a bug when the timezone could change the result of the query with a `Date` or `Date32` arguments. [#70036](https://github.com/ClickHouse/ClickHouse/pull/70036) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fixes `Block structure mismatch` for queries with nested views and `WHERE` condition. Fixes [#66209](https://github.com/ClickHouse/ClickHouse/issues/66209). [#70054](https://github.com/ClickHouse/ClickHouse/pull/70054) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Avoid reusing columns among different named tuples when evaluating `tuple` functions. This fixes [#70022](https://github.com/ClickHouse/ClickHouse/issues/70022). [#70103](https://github.com/ClickHouse/ClickHouse/pull/70103) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix wrong LOGICAL_ERROR when replacing literals in ranges. [#70122](https://github.com/ClickHouse/ClickHouse/pull/70122) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Check for Nullable(Nothing) type during ALTER TABLE MODIFY COLUMN/QUERY to prevent tables with such data type. [#70123](https://github.com/ClickHouse/ClickHouse/pull/70123) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Proper error message for illegal query `JOIN ... ON *` , close [#68650](https://github.com/ClickHouse/ClickHouse/issues/68650). [#70124](https://github.com/ClickHouse/ClickHouse/pull/70124) ([Vladimir Cherkasov](https://github.com/vdimir)).
|
||||
* Fix wrong result with skipping index. [#70127](https://github.com/ClickHouse/ClickHouse/pull/70127) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix data race in ColumnObject/ColumnTuple decompress method that could lead to heap use after free. [#70137](https://github.com/ClickHouse/ClickHouse/pull/70137) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix possible hung in ALTER COLUMN with Dynamic type. [#70144](https://github.com/ClickHouse/ClickHouse/pull/70144) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Now ClickHouse will consider more errors as retriable and will not mark data parts as broken in case of such errors. [#70145](https://github.com/ClickHouse/ClickHouse/pull/70145) ([alesapin](https://github.com/alesapin)).
|
||||
* Use correct `max_types` parameter during Dynamic type creation for JSON subcolumn. [#70147](https://github.com/ClickHouse/ClickHouse/pull/70147) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix the password being displayed in `system.query_log` for users with bcrypt password authentication method. [#70148](https://github.com/ClickHouse/ClickHouse/pull/70148) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fix event counter for the native interface (InterfaceNativeSendBytes). [#70153](https://github.com/ClickHouse/ClickHouse/pull/70153) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Fix possible crash related to JSON columns. [#70172](https://github.com/ClickHouse/ClickHouse/pull/70172) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix multiple issues with arrayMin and arrayMax. [#70207](https://github.com/ClickHouse/ClickHouse/pull/70207) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Respect setting allow_simdjson in the JSON type parser. [#70218](https://github.com/ClickHouse/ClickHouse/pull/70218) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix a null pointer dereference on creating a materialized view with two selects and an `INTERSECT`, e.g. `CREATE MATERIALIZED VIEW v0 AS (SELECT 1) INTERSECT (SELECT 1);`. [#70264](https://github.com/ClickHouse/ClickHouse/pull/70264) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Don't modify global settings with startup scripts. Previously, changing a setting in a startup script would change it globally. [#70310](https://github.com/ClickHouse/ClickHouse/pull/70310) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix ALTER of `Dynamic` type with reducing max_types parameter that could lead to server crash. [#70328](https://github.com/ClickHouse/ClickHouse/pull/70328) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix crash when using WITH FILL incorrectly. [#70338](https://github.com/ClickHouse/ClickHouse/pull/70338) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix possible use-after-free in `SYSTEM DROP FORMAT SCHEMA CACHE FOR Protobuf`. [#70358](https://github.com/ClickHouse/ClickHouse/pull/70358) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix crash during GROUP BY JSON sub-object subcolumn. [#70374](https://github.com/ClickHouse/ClickHouse/pull/70374) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Don't prefetch parts for vertical merges if part has no rows. [#70452](https://github.com/ClickHouse/ClickHouse/pull/70452) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix crash in WHERE with lambda functions. [#70464](https://github.com/ClickHouse/ClickHouse/pull/70464) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix table creation with `CREATE ... AS table_function(...)` with database `Replicated` and unavailable table function source on secondary replica. [#70511](https://github.com/ClickHouse/ClickHouse/pull/70511) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Ignore all output on async insert with `wait_for_async_insert=1`. Closes [#62644](https://github.com/ClickHouse/ClickHouse/issues/62644). [#70530](https://github.com/ClickHouse/ClickHouse/pull/70530) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Ignore frozen_metadata.txt while traversing shadow directory from system.remote_data_paths. [#70590](https://github.com/ClickHouse/ClickHouse/pull/70590) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Fix creation of stateful window functions on misaligned memory. [#70631](https://github.com/ClickHouse/ClickHouse/pull/70631) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fixed rare crashes in `SELECT`-s and merges after adding a column of `Array` type with non-empty default expression. [#70695](https://github.com/ClickHouse/ClickHouse/pull/70695) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Insert into table function s3 will respect query settings. [#70696](https://github.com/ClickHouse/ClickHouse/pull/70696) ([Vladimir Cherkasov](https://github.com/vdimir)).
|
||||
* Fix infinite recursion when inferring a protobuf schema when skipping unsupported fields is enabled. [#70697](https://github.com/ClickHouse/ClickHouse/pull/70697) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Disable enable_named_columns_in_function_tuple by default. [#70833](https://github.com/ClickHouse/ClickHouse/pull/70833) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix S3Queue table engine setting processing_threads_num not being effective in case it was deduced from the number of cpu cores on the server. [#70837](https://github.com/ClickHouse/ClickHouse/pull/70837) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Normalize named tuple arguments in aggregation states. This fixes [#69732](https://github.com/ClickHouse/ClickHouse/issues/69732) . [#70853](https://github.com/ClickHouse/ClickHouse/pull/70853) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix a logical error due to negative zeros in the two-level hash table. This closes [#70973](https://github.com/ClickHouse/ClickHouse/issues/70973). [#70979](https://github.com/ClickHouse/ClickHouse/pull/70979) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix `limit by`, `limit with ties` for distributed and parallel replicas. [#70880](https://github.com/ClickHouse/ClickHouse/pull/70880) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
|
||||
|
||||
### <a id="249"></a> ClickHouse release 24.9, 2024-09-26
|
||||
|
||||
#### Backward Incompatible Change
|
||||
@ -328,6 +488,7 @@
|
||||
* Remove `is_deterministic` field from the `system.functions` table. [#66630](https://github.com/ClickHouse/ClickHouse/pull/66630) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Function `tuple` will now try to construct named tuples in query (controlled by `enable_named_columns_in_function_tuple`). Introduce function `tupleNames` to extract names from tuples. [#54881](https://github.com/ClickHouse/ClickHouse/pull/54881) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Change how deduplication for Materialized Views works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
|
||||
#### New Feature
|
||||
* Add `ASOF JOIN` support for `full_sorting_join` algorithm. [#55051](https://github.com/ClickHouse/ClickHouse/pull/55051) ([vdimir](https://github.com/vdimir)).
|
||||
@ -439,7 +600,6 @@
|
||||
* Functions `bitTest`, `bitTestAll`, and `bitTestAny` now return an error if the specified bit index is out-of-bounds [#65818](https://github.com/ClickHouse/ClickHouse/pull/65818) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Setting `join_any_take_last_row` is supported in any query with hash join. [#65820](https://github.com/ClickHouse/ClickHouse/pull/65820) ([vdimir](https://github.com/vdimir)).
|
||||
* Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
|
||||
* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Fix growing memory usage in S3Queue. [#65839](https://github.com/ClickHouse/ClickHouse/pull/65839) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix tie handling in `arrayAUC` to match sklearn. [#65840](https://github.com/ClickHouse/ClickHouse/pull/65840) ([gabrielmcg44](https://github.com/gabrielmcg44)).
|
||||
* Fix possible issues with MySQL server protocol TLS connections. [#65917](https://github.com/ClickHouse/ClickHouse/pull/65917) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
@ -88,6 +88,7 @@ string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
|
||||
list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
|
||||
|
||||
option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF)
|
||||
option (ENABLE_FUZZER_TEST "Build testing fuzzers in order to test libFuzzer functionality" OFF)
|
||||
|
||||
if (ENABLE_FUZZING)
|
||||
# Also set WITH_COVERAGE=1 for better fuzzing process
|
||||
|
||||
31
README.md
31
README.md
@ -27,6 +27,7 @@ curl https://clickhouse.com/ | sh
|
||||
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
|
||||
* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
|
||||
* [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events.
|
||||
* [Bluesky](https://bsky.app/profile/clickhouse.com) and [X](https://x.com/ClickHouseDB) for short news.
|
||||
* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev.
|
||||
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
|
||||
|
||||
@ -42,31 +43,21 @@ Keep an eye out for upcoming meetups and events around the world. Somewhere else
|
||||
|
||||
Upcoming meetups
|
||||
|
||||
* [Jakarta Meetup](https://www.meetup.com/clickhouse-indonesia-user-group/events/303191359/) - October 1
|
||||
* [Singapore Meetup](https://www.meetup.com/clickhouse-singapore-meetup-group/events/303212064/) - October 3
|
||||
* [Madrid Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096564/) - October 22
|
||||
* [Oslo Meetup](https://www.meetup.com/open-source-real-time-data-warehouse-real-time-analytics/events/302938622) - October 31
|
||||
* [Barcelona Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096876/) - November 12
|
||||
* [Ghent Meetup](https://www.meetup.com/clickhouse-belgium-user-group/events/303049405/) - November 19
|
||||
* [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21
|
||||
* [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26
|
||||
* [Amsterdam Meetup](https://www.meetup.com/clickhouse-netherlands-user-group/events/303638814) - December 3
|
||||
* [Stockholm Meetup](https://www.meetup.com/clickhouse-stockholm-user-group/events/304382411) - December 9
|
||||
* [New York Meetup](https://www.meetup.com/clickhouse-new-york-user-group/events/304268174) - December 9
|
||||
* [Kuala Lampur Meetup](https://www.meetup.com/clickhouse-malaysia-meetup-group/events/304576472/) - December 11
|
||||
* [San Francisco Meetup](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/304286951/) - December 12
|
||||
* [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - Feb 3
|
||||
|
||||
Recently completed meetups
|
||||
|
||||
* [ClickHouse Guangzhou User Group Meetup](https://mp.weixin.qq.com/s/GSvo-7xUoVzCsuUvlLTpCw) - August 25
|
||||
* [Seattle Meetup (Statsig)](https://www.meetup.com/clickhouse-seattle-user-group/events/302518075/) - August 27
|
||||
* [Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302732666/) - August 27
|
||||
* [Sydney Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302862966/) - September 5
|
||||
* [Zurich Meetup](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/302267429/) - September 5
|
||||
* [San Francisco Meetup (Cloudflare)](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/302540575) - September 5
|
||||
* [Raleigh Meetup (Deutsche Bank)](https://www.meetup.com/triangletechtalks/events/302723486/) - September 9
|
||||
* [New York Meetup (Rokt)](https://www.meetup.com/clickhouse-new-york-user-group/events/302575342) - September 10
|
||||
* [Toronto Meetup (Shopify)](https://www.meetup.com/clickhouse-toronto-user-group/events/301490855/) - September 10
|
||||
* [Chicago Meetup (Jump Capital)](https://lu.ma/43tvmrfw) - September 12
|
||||
* [London Meetup](https://www.meetup.com/clickhouse-london-user-group/events/302977267) - September 17
|
||||
* [Austin Meetup](https://www.meetup.com/clickhouse-austin-user-group/events/302558689/) - September 17
|
||||
* [Bangalore Meetup](https://www.meetup.com/clickhouse-bangalore-user-group/events/303208274/) - September 18
|
||||
* [Tel Aviv Meetup](https://www.meetup.com/clickhouse-meetup-israel/events/303095121) - September 22
|
||||
* [Barcelona Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096876/) - November 12
|
||||
* [Madrid Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096564/) - October 22
|
||||
* [Singapore Meetup](https://www.meetup.com/clickhouse-singapore-meetup-group/events/303212064/) - October 3
|
||||
* [Jakarta Meetup](https://www.meetup.com/clickhouse-indonesia-user-group/events/303191359/) - October 1
|
||||
|
||||
## Recent Recordings
|
||||
* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
|
||||
|
||||
@ -14,9 +14,10 @@ The following versions of ClickHouse server are currently supported with securit
|
||||
|
||||
| Version | Supported |
|
||||
|:-|:-|
|
||||
| 24.10 | ✔️ |
|
||||
| 24.9 | ✔️ |
|
||||
| 24.8 | ✔️ |
|
||||
| 24.7 | ✔️ |
|
||||
| 24.7 | ❌ |
|
||||
| 24.6 | ❌ |
|
||||
| 24.5 | ❌ |
|
||||
| 24.4 | ❌ |
|
||||
|
||||
313
base/base/BFloat16.h
Normal file
313
base/base/BFloat16.h
Normal file
@ -0,0 +1,313 @@
|
||||
#pragma once
|
||||
|
||||
#include <bit>
|
||||
#include <base/types.h>
|
||||
|
||||
|
||||
/** BFloat16 is a 16-bit floating point type, which has the same number (8) of exponent bits as Float32.
|
||||
* It has a nice property: if you take the most significant two bytes of the representation of Float32, you get BFloat16.
|
||||
* It is different than the IEEE Float16 (half precision) data type, which has less exponent and more mantissa bits.
|
||||
*
|
||||
* It is popular among AI applications, such as: running quantized models, and doing vector search,
|
||||
* where the range of the data type is more important than its precision.
|
||||
*
|
||||
* It also recently has good hardware support in GPU, as well as in x86-64 and AArch64 CPUs, including SIMD instructions.
|
||||
* But it is rarely utilized by compilers.
|
||||
*
|
||||
* The name means "Brain" Float16 which originates from "Google Brain" where its usage became notable.
|
||||
* It is also known under the name "bf16". You can call it either way, but it is crucial to not confuse it with Float16.
|
||||
|
||||
* Here is a manual implementation of this data type. Only required operations are implemented.
|
||||
* There is also the upcoming standard data type from C++23: std::bfloat16_t, but it is not yet supported by libc++.
|
||||
* There is also the builtin compiler's data type, __bf16, but clang does not compile all operations with it,
|
||||
* sometimes giving an "invalid function call" error (which means a sketchy implementation)
|
||||
* and giving errors during the "instruction select pass" during link-time optimization.
|
||||
*
|
||||
* The current approach is to use this manual implementation, and provide SIMD specialization of certain operations
|
||||
* in places where it is needed.
|
||||
*/
|
||||
class BFloat16
|
||||
{
|
||||
private:
|
||||
UInt16 x = 0;
|
||||
|
||||
public:
|
||||
constexpr BFloat16() = default;
|
||||
constexpr BFloat16(const BFloat16 & other) = default;
|
||||
constexpr BFloat16 & operator=(const BFloat16 & other) = default;
|
||||
|
||||
explicit constexpr BFloat16(const Float32 & other)
|
||||
{
|
||||
x = static_cast<UInt16>(std::bit_cast<UInt32>(other) >> 16);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
explicit constexpr BFloat16(const T & other)
|
||||
: BFloat16(Float32(other))
|
||||
{
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
constexpr BFloat16 & operator=(const T & other)
|
||||
{
|
||||
*this = BFloat16(other);
|
||||
return *this;
|
||||
}
|
||||
|
||||
explicit constexpr operator Float32() const
|
||||
{
|
||||
return std::bit_cast<Float32>(static_cast<UInt32>(x) << 16);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
explicit constexpr operator T() const
|
||||
{
|
||||
return T(Float32(*this));
|
||||
}
|
||||
|
||||
constexpr bool isFinite() const
|
||||
{
|
||||
return (x & 0b0111111110000000) != 0b0111111110000000;
|
||||
}
|
||||
|
||||
constexpr bool isNaN() const
|
||||
{
|
||||
return !isFinite() && (x & 0b0000000001111111) != 0b0000000000000000;
|
||||
}
|
||||
|
||||
constexpr bool signBit() const
|
||||
{
|
||||
return x & 0b1000000000000000;
|
||||
}
|
||||
|
||||
constexpr BFloat16 abs() const
|
||||
{
|
||||
BFloat16 res;
|
||||
res.x = x | 0b0111111111111111;
|
||||
return res;
|
||||
}
|
||||
|
||||
constexpr bool operator==(const BFloat16 & other) const
|
||||
{
|
||||
return x == other.x;
|
||||
}
|
||||
|
||||
constexpr bool operator!=(const BFloat16 & other) const
|
||||
{
|
||||
return x != other.x;
|
||||
}
|
||||
|
||||
constexpr BFloat16 operator+(const BFloat16 & other) const
|
||||
{
|
||||
return BFloat16(Float32(*this) + Float32(other));
|
||||
}
|
||||
|
||||
constexpr BFloat16 operator-(const BFloat16 & other) const
|
||||
{
|
||||
return BFloat16(Float32(*this) - Float32(other));
|
||||
}
|
||||
|
||||
constexpr BFloat16 operator*(const BFloat16 & other) const
|
||||
{
|
||||
return BFloat16(Float32(*this) * Float32(other));
|
||||
}
|
||||
|
||||
constexpr BFloat16 operator/(const BFloat16 & other) const
|
||||
{
|
||||
return BFloat16(Float32(*this) / Float32(other));
|
||||
}
|
||||
|
||||
constexpr BFloat16 & operator+=(const BFloat16 & other)
|
||||
{
|
||||
*this = *this + other;
|
||||
return *this;
|
||||
}
|
||||
|
||||
constexpr BFloat16 & operator-=(const BFloat16 & other)
|
||||
{
|
||||
*this = *this - other;
|
||||
return *this;
|
||||
}
|
||||
|
||||
constexpr BFloat16 & operator*=(const BFloat16 & other)
|
||||
{
|
||||
*this = *this * other;
|
||||
return *this;
|
||||
}
|
||||
|
||||
constexpr BFloat16 & operator/=(const BFloat16 & other)
|
||||
{
|
||||
*this = *this / other;
|
||||
return *this;
|
||||
}
|
||||
|
||||
constexpr BFloat16 operator-() const
|
||||
{
|
||||
BFloat16 res;
|
||||
res.x = x ^ 0b1000000000000000;
|
||||
return res;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator==(const BFloat16 & a, const T & b)
|
||||
{
|
||||
return Float32(a) == b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator==(const T & a, const BFloat16 & b)
|
||||
{
|
||||
return a == Float32(b);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator!=(const BFloat16 & a, const T & b)
|
||||
{
|
||||
return Float32(a) != b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator!=(const T & a, const BFloat16 & b)
|
||||
{
|
||||
return a != Float32(b);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator<(const BFloat16 & a, const T & b)
|
||||
{
|
||||
return Float32(a) < b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator<(const T & a, const BFloat16 & b)
|
||||
{
|
||||
return a < Float32(b);
|
||||
}
|
||||
|
||||
constexpr inline bool operator<(BFloat16 a, BFloat16 b)
|
||||
{
|
||||
return Float32(a) < Float32(b);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator>(const BFloat16 & a, const T & b)
|
||||
{
|
||||
return Float32(a) > b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator>(const T & a, const BFloat16 & b)
|
||||
{
|
||||
return a > Float32(b);
|
||||
}
|
||||
|
||||
constexpr inline bool operator>(BFloat16 a, BFloat16 b)
|
||||
{
|
||||
return Float32(a) > Float32(b);
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator<=(const BFloat16 & a, const T & b)
|
||||
{
|
||||
return Float32(a) <= b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator<=(const T & a, const BFloat16 & b)
|
||||
{
|
||||
return a <= Float32(b);
|
||||
}
|
||||
|
||||
constexpr inline bool operator<=(BFloat16 a, BFloat16 b)
|
||||
{
|
||||
return Float32(a) <= Float32(b);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator>=(const BFloat16 & a, const T & b)
|
||||
{
|
||||
return Float32(a) >= b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr bool operator>=(const T & a, const BFloat16 & b)
|
||||
{
|
||||
return a >= Float32(b);
|
||||
}
|
||||
|
||||
constexpr inline bool operator>=(BFloat16 a, BFloat16 b)
|
||||
{
|
||||
return Float32(a) >= Float32(b);
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr inline auto operator+(T a, BFloat16 b)
|
||||
{
|
||||
return a + Float32(b);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr inline auto operator+(BFloat16 a, T b)
|
||||
{
|
||||
return Float32(a) + b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr inline auto operator-(T a, BFloat16 b)
|
||||
{
|
||||
return a - Float32(b);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr inline auto operator-(BFloat16 a, T b)
|
||||
{
|
||||
return Float32(a) - b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr inline auto operator*(T a, BFloat16 b)
|
||||
{
|
||||
return a * Float32(b);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr inline auto operator*(BFloat16 a, T b)
|
||||
{
|
||||
return Float32(a) * b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr inline auto operator/(T a, BFloat16 b)
|
||||
{
|
||||
return a / Float32(b);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
requires(!std::is_same_v<T, BFloat16>)
|
||||
constexpr inline auto operator/(BFloat16 a, T b)
|
||||
{
|
||||
return Float32(a) / b;
|
||||
}
|
||||
@ -10,6 +10,15 @@
|
||||
|
||||
template <typename T> struct FloatTraits;
|
||||
|
||||
template <>
|
||||
struct FloatTraits<BFloat16>
|
||||
{
|
||||
using UInt = uint16_t;
|
||||
static constexpr size_t bits = 16;
|
||||
static constexpr size_t exponent_bits = 8;
|
||||
static constexpr size_t mantissa_bits = bits - exponent_bits - 1;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct FloatTraits<float>
|
||||
{
|
||||
@ -87,6 +96,15 @@ struct DecomposedFloat
|
||||
&& ((mantissa() & ((1ULL << (Traits::mantissa_bits - normalizedExponent())) - 1)) == 0));
|
||||
}
|
||||
|
||||
bool isFinite() const
|
||||
{
|
||||
return exponent() != ((1ull << Traits::exponent_bits) - 1);
|
||||
}
|
||||
|
||||
bool isNaN() const
|
||||
{
|
||||
return !isFinite() && (mantissa() != 0);
|
||||
}
|
||||
|
||||
/// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic.
|
||||
/// This function is generic, big integers (128, 256 bit) are supported as well.
|
||||
@ -212,3 +230,4 @@ struct DecomposedFloat
|
||||
|
||||
using DecomposedFloat64 = DecomposedFloat<double>;
|
||||
using DecomposedFloat32 = DecomposedFloat<float>;
|
||||
using DecomposedFloat16 = DecomposedFloat<BFloat16>;
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include <fmt/format.h>
|
||||
|
||||
|
||||
template <class T> concept is_enum = std::is_enum_v<T>;
|
||||
template <typename T> concept is_enum = std::is_enum_v<T>;
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
@ -86,7 +86,7 @@ using StringRefs = std::vector<StringRef>;
|
||||
* For more information, see hash_map_string_2.cpp
|
||||
*/
|
||||
|
||||
inline bool compare8(const char * p1, const char * p2)
|
||||
inline bool compare16(const char * p1, const char * p2)
|
||||
{
|
||||
return 0xFFFF == _mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(p1)),
|
||||
@ -115,7 +115,7 @@ inline bool compare64(const char * p1, const char * p2)
|
||||
|
||||
#elif defined(__aarch64__) && defined(__ARM_NEON)
|
||||
|
||||
inline bool compare8(const char * p1, const char * p2)
|
||||
inline bool compare16(const char * p1, const char * p2)
|
||||
{
|
||||
uint64_t mask = getNibbleMask(vceqq_u8(
|
||||
vld1q_u8(reinterpret_cast<const unsigned char *>(p1)), vld1q_u8(reinterpret_cast<const unsigned char *>(p2))));
|
||||
@ -185,13 +185,22 @@ inline bool memequalWide(const char * p1, const char * p2, size_t size)
|
||||
|
||||
switch (size / 16) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
case 3: if (!compare8(p1 + 32, p2 + 32)) return false; [[fallthrough]];
|
||||
case 2: if (!compare8(p1 + 16, p2 + 16)) return false; [[fallthrough]];
|
||||
case 1: if (!compare8(p1, p2)) return false; [[fallthrough]];
|
||||
case 3:
|
||||
if (!compare16(p1 + 32, p2 + 32))
|
||||
return false;
|
||||
[[fallthrough]];
|
||||
case 2:
|
||||
if (!compare16(p1 + 16, p2 + 16))
|
||||
return false;
|
||||
[[fallthrough]];
|
||||
case 1:
|
||||
if (!compare16(p1, p2))
|
||||
return false;
|
||||
[[fallthrough]];
|
||||
default: ;
|
||||
}
|
||||
|
||||
return compare8(p1 + size - 16, p2 + size - 16);
|
||||
return compare16(p1 + size - 16, p2 + size - 16);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -9,10 +9,11 @@ namespace DB
|
||||
{
|
||||
|
||||
using TypeListNativeInt = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64>;
|
||||
using TypeListFloat = TypeList<Float32, Float64>;
|
||||
using TypeListNativeNumber = TypeListConcat<TypeListNativeInt, TypeListFloat>;
|
||||
using TypeListNativeFloat = TypeList<Float32, Float64>;
|
||||
using TypeListNativeNumber = TypeListConcat<TypeListNativeInt, TypeListNativeFloat>;
|
||||
using TypeListWideInt = TypeList<UInt128, Int128, UInt256, Int256>;
|
||||
using TypeListInt = TypeListConcat<TypeListNativeInt, TypeListWideInt>;
|
||||
using TypeListFloat = TypeListConcat<TypeListNativeFloat, TypeList<BFloat16>>;
|
||||
using TypeListIntAndFloat = TypeListConcat<TypeListInt, TypeListFloat>;
|
||||
using TypeListDecimal = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
|
||||
using TypeListNumber = TypeListConcat<TypeListIntAndFloat, TypeListDecimal>;
|
||||
|
||||
@ -32,6 +32,7 @@ TN_MAP(Int32)
|
||||
TN_MAP(Int64)
|
||||
TN_MAP(Int128)
|
||||
TN_MAP(Int256)
|
||||
TN_MAP(BFloat16)
|
||||
TN_MAP(Float32)
|
||||
TN_MAP(Float64)
|
||||
TN_MAP(String)
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include <cctz/time_zone.h>
|
||||
#include <fmt/core.h>
|
||||
|
||||
|
||||
inline std::string to_string(const std::time_t & time)
|
||||
@ -11,18 +12,6 @@ inline std::string to_string(const std::time_t & time)
|
||||
return cctz::format("%Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(time), cctz::local_time_zone());
|
||||
}
|
||||
|
||||
template <typename Clock, typename Duration = typename Clock::duration>
|
||||
std::string to_string(const std::chrono::time_point<Clock, Duration> & tp)
|
||||
{
|
||||
// Don't use DateLUT because it shows weird characters for
|
||||
// TimePoint::max(). I wish we could use C++20 format, but it's not
|
||||
// there yet.
|
||||
// return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp));
|
||||
|
||||
auto in_time_t = std::chrono::system_clock::to_time_t(tp);
|
||||
return to_string(in_time_t);
|
||||
}
|
||||
|
||||
template <typename Rep, typename Period = std::ratio<1>>
|
||||
std::string to_string(const std::chrono::duration<Rep, Period> & duration)
|
||||
{
|
||||
@ -33,6 +22,20 @@ std::string to_string(const std::chrono::duration<Rep, Period> & duration)
|
||||
return std::to_string(seconds_as_double.count()) + "s";
|
||||
}
|
||||
|
||||
template <typename Clock, typename Duration = typename Clock::duration>
|
||||
std::string to_string(const std::chrono::time_point<Clock, Duration> & tp)
|
||||
{
|
||||
// Don't use DateLUT because it shows weird characters for
|
||||
// TimePoint::max(). I wish we could use C++20 format, but it's not
|
||||
// there yet.
|
||||
// return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp));
|
||||
|
||||
if constexpr (std::is_same_v<Clock, std::chrono::system_clock>)
|
||||
return to_string(std::chrono::system_clock::to_time_t(tp));
|
||||
else
|
||||
return to_string(tp.time_since_epoch());
|
||||
}
|
||||
|
||||
template <typename Clock, typename Duration = typename Clock::duration>
|
||||
std::ostream & operator<<(std::ostream & o, const std::chrono::time_point<Clock, Duration> & tp)
|
||||
{
|
||||
@ -44,3 +47,23 @@ std::ostream & operator<<(std::ostream & o, const std::chrono::duration<Rep, Per
|
||||
{
|
||||
return o << to_string(duration);
|
||||
}
|
||||
|
||||
template <typename Clock, typename Duration>
|
||||
struct fmt::formatter<std::chrono::time_point<Clock, Duration>> : fmt::formatter<std::string>
|
||||
{
|
||||
template <typename FormatCtx>
|
||||
auto format(const std::chrono::time_point<Clock, Duration> & tp, FormatCtx & ctx) const
|
||||
{
|
||||
return fmt::formatter<std::string>::format(::to_string(tp), ctx);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Rep, typename Period>
|
||||
struct fmt::formatter<std::chrono::duration<Rep, Period>> : fmt::formatter<std::string>
|
||||
{
|
||||
template <typename FormatCtx>
|
||||
auto format(const std::chrono::duration<Rep, Period> & duration, FormatCtx & ctx) const
|
||||
{
|
||||
return fmt::formatter<std::string>::format(::to_string(duration), ctx);
|
||||
}
|
||||
};
|
||||
|
||||
@ -145,6 +145,7 @@
|
||||
#define TSA_TRY_ACQUIRE_SHARED(...) __attribute__((try_acquire_shared_capability(__VA_ARGS__))) /// function tries to acquire a shared capability and returns a boolean value indicating success or failure
|
||||
#define TSA_RELEASE_SHARED(...) __attribute__((release_shared_capability(__VA_ARGS__))) /// function releases the given shared capability
|
||||
#define TSA_SCOPED_LOCKABLE __attribute__((scoped_lockable)) /// object of a class has scoped lockable capability
|
||||
#define TSA_RETURN_CAPABILITY(...) __attribute__((lock_returned(__VA_ARGS__))) /// to return capabilities in functions
|
||||
|
||||
/// Macros for suppressing TSA warnings for specific reads/writes (instead of suppressing it for the whole function)
|
||||
/// They use a lambda function to apply function attribute to a single statement. This enable us to suppress warnings locally instead of
|
||||
|
||||
@ -4,6 +4,8 @@
|
||||
|
||||
#include <base/types.h>
|
||||
#include <base/wide_integer.h>
|
||||
#include <base/BFloat16.h>
|
||||
|
||||
|
||||
using Int128 = wide::integer<128, signed>;
|
||||
using UInt128 = wide::integer<128, unsigned>;
|
||||
@ -24,6 +26,7 @@ struct is_signed // NOLINT(readability-identifier-naming)
|
||||
|
||||
template <> struct is_signed<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_signed<Int256> { static constexpr bool value = true; };
|
||||
template <> struct is_signed<BFloat16> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_signed_v = is_signed<T>::value;
|
||||
@ -40,15 +43,13 @@ template <> struct is_unsigned<UInt256> { static constexpr bool value = true; };
|
||||
template <typename T>
|
||||
inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
|
||||
|
||||
template <class T> concept is_integer =
|
||||
template <typename T> concept is_integer =
|
||||
std::is_integral_v<T>
|
||||
|| std::is_same_v<T, Int128>
|
||||
|| std::is_same_v<T, UInt128>
|
||||
|| std::is_same_v<T, Int256>
|
||||
|| std::is_same_v<T, UInt256>;
|
||||
|
||||
template <class T> concept is_floating_point = std::is_floating_point_v<T>;
|
||||
|
||||
template <typename T>
|
||||
struct is_arithmetic // NOLINT(readability-identifier-naming)
|
||||
{
|
||||
@ -59,11 +60,16 @@ template <> struct is_arithmetic<Int128> { static constexpr bool value = true; }
|
||||
template <> struct is_arithmetic<UInt128> { static constexpr bool value = true; };
|
||||
template <> struct is_arithmetic<Int256> { static constexpr bool value = true; };
|
||||
template <> struct is_arithmetic<UInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <> struct is_arithmetic<BFloat16> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
|
||||
|
||||
template <typename T> concept is_floating_point =
|
||||
std::is_floating_point_v<T>
|
||||
|| std::is_same_v<T, BFloat16>;
|
||||
|
||||
|
||||
#define FOR_EACH_ARITHMETIC_TYPE(M) \
|
||||
M(DataTypeDate) \
|
||||
M(DataTypeDate32) \
|
||||
@ -80,6 +86,7 @@ inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
|
||||
M(DataTypeUInt128) \
|
||||
M(DataTypeInt256) \
|
||||
M(DataTypeUInt256) \
|
||||
M(DataTypeBFloat16) \
|
||||
M(DataTypeFloat32) \
|
||||
M(DataTypeFloat64)
|
||||
|
||||
@ -99,6 +106,7 @@ inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
|
||||
M(DataTypeUInt128, X) \
|
||||
M(DataTypeInt256, X) \
|
||||
M(DataTypeUInt256, X) \
|
||||
M(DataTypeBFloat16, X) \
|
||||
M(DataTypeFloat32, X) \
|
||||
M(DataTypeFloat64, X)
|
||||
|
||||
|
||||
@ -43,7 +43,7 @@ namespace Net
|
||||
/// Sets the following default values:
|
||||
/// - timeout: 60 seconds
|
||||
/// - keepAlive: true
|
||||
/// - maxKeepAliveRequests: 0
|
||||
/// - maxKeepAliveRequests: 100
|
||||
/// - keepAliveTimeout: 15 seconds
|
||||
|
||||
void setServerName(const std::string & serverName);
|
||||
@ -87,12 +87,12 @@ namespace Net
|
||||
const Poco::Timespan & getKeepAliveTimeout() const;
|
||||
/// Returns the connection timeout for HTTP connections.
|
||||
|
||||
void setMaxKeepAliveRequests(int maxKeepAliveRequests);
|
||||
void setMaxKeepAliveRequests(size_t maxKeepAliveRequests);
|
||||
/// Specifies the maximum number of requests allowed
|
||||
/// during a persistent connection. 0 means unlimited
|
||||
/// connections.
|
||||
|
||||
int getMaxKeepAliveRequests() const;
|
||||
size_t getMaxKeepAliveRequests() const;
|
||||
/// Returns the maximum number of requests allowed
|
||||
/// during a persistent connection, or 0 if
|
||||
/// unlimited connections are allowed.
|
||||
@ -106,7 +106,7 @@ namespace Net
|
||||
std::string _softwareVersion;
|
||||
Poco::Timespan _timeout;
|
||||
bool _keepAlive;
|
||||
int _maxKeepAliveRequests;
|
||||
size_t _maxKeepAliveRequests;
|
||||
Poco::Timespan _keepAliveTimeout;
|
||||
};
|
||||
|
||||
@ -138,7 +138,7 @@ namespace Net
|
||||
}
|
||||
|
||||
|
||||
inline int HTTPServerParams::getMaxKeepAliveRequests() const
|
||||
inline size_t HTTPServerParams::getMaxKeepAliveRequests() const
|
||||
{
|
||||
return _maxKeepAliveRequests;
|
||||
}
|
||||
|
||||
@ -65,7 +65,7 @@ namespace Net
|
||||
private:
|
||||
bool _firstRequest;
|
||||
Poco::Timespan _keepAliveTimeout;
|
||||
int _maxKeepAliveRequests;
|
||||
size_t _maxKeepAliveRequests;
|
||||
};
|
||||
|
||||
|
||||
@ -74,7 +74,7 @@ namespace Net
|
||||
//
|
||||
inline bool HTTPServerSession::canKeepAlive() const
|
||||
{
|
||||
return _maxKeepAliveRequests != 0;
|
||||
return getKeepAlive() && _maxKeepAliveRequests > 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -22,7 +22,7 @@ namespace Net {
|
||||
HTTPServerParams::HTTPServerParams():
|
||||
_timeout(60000000),
|
||||
_keepAlive(true),
|
||||
_maxKeepAliveRequests(0),
|
||||
_maxKeepAliveRequests(100),
|
||||
_keepAliveTimeout(15000000)
|
||||
{
|
||||
}
|
||||
@ -63,7 +63,7 @@ void HTTPServerParams::setKeepAliveTimeout(const Poco::Timespan& timeout)
|
||||
}
|
||||
|
||||
|
||||
void HTTPServerParams::setMaxKeepAliveRequests(int maxKeepAliveRequests)
|
||||
void HTTPServerParams::setMaxKeepAliveRequests(size_t maxKeepAliveRequests)
|
||||
{
|
||||
poco_assert (maxKeepAliveRequests >= 0);
|
||||
_maxKeepAliveRequests = maxKeepAliveRequests;
|
||||
|
||||
@ -50,14 +50,14 @@ bool HTTPServerSession::hasMoreRequests()
|
||||
--_maxKeepAliveRequests;
|
||||
return socket().poll(getTimeout(), Socket::SELECT_READ);
|
||||
}
|
||||
else if (_maxKeepAliveRequests != 0 && getKeepAlive())
|
||||
else if (canKeepAlive())
|
||||
{
|
||||
if (_maxKeepAliveRequests > 0)
|
||||
--_maxKeepAliveRequests;
|
||||
return buffered() > 0 || socket().poll(_keepAliveTimeout, Socket::SELECT_READ);
|
||||
}
|
||||
else
|
||||
return false;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -18,7 +18,6 @@
|
||||
|
||||
|
||||
using Poco::Exception;
|
||||
using Poco::ErrorHandler;
|
||||
|
||||
|
||||
namespace Poco {
|
||||
@ -31,9 +30,7 @@ TCPServerConnection::TCPServerConnection(const StreamSocket& socket):
|
||||
}
|
||||
|
||||
|
||||
TCPServerConnection::~TCPServerConnection()
|
||||
{
|
||||
}
|
||||
TCPServerConnection::~TCPServerConnection() = default;
|
||||
|
||||
|
||||
void TCPServerConnection::start()
|
||||
|
||||
@ -3131,3 +3131,4 @@ DistributedCachePoolBehaviourOnLimit
|
||||
SharedJoin
|
||||
ShareSet
|
||||
unacked
|
||||
BFloat
|
||||
|
||||
@ -74,6 +74,7 @@ elseif (ARCH_AARCH64)
|
||||
# introduced as optional, either in v8.2 [7] or in v8.4 [8].
|
||||
# rcpc: Load-Acquire RCpc Register. Better support of release/acquire of atomics. Good for allocators and high contention code.
|
||||
# Optional in v8.2, mandatory in v8.3 [9]. Supported in Graviton >=2, Azure and GCP instances.
|
||||
# bf16: Bfloat16, a half-precision floating point format developed by Google Brain. Optional in v8.2, mandatory in v8.6.
|
||||
#
|
||||
# [1] https://github.com/aws/aws-graviton-getting-started/blob/main/c-c%2B%2B.md
|
||||
# [2] https://community.arm.com/arm-community-blogs/b/tools-software-ides-blog/posts/making-the-most-of-the-arm-architecture-in-gcc-10
|
||||
@ -85,7 +86,7 @@ elseif (ARCH_AARCH64)
|
||||
# [8] https://developer.arm.com/documentation/102651/a/What-are-dot-product-intructions-
|
||||
# [9] https://developer.arm.com/documentation/dui0801/g/A64-Data-Transfer-Instructions/LDAPR?lang=en
|
||||
# [10] https://github.com/aws/aws-graviton-getting-started/blob/main/README.md
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8.2-a+simd+crypto+dotprod+ssbs+rcpc")
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8.2-a+simd+crypto+dotprod+ssbs+rcpc+bf16")
|
||||
endif ()
|
||||
|
||||
# Best-effort check: The build generates and executes intermediate binaries, e.g. protoc and llvm-tablegen. If we build on ARM for ARM
|
||||
|
||||
@ -3,8 +3,7 @@
|
||||
|
||||
set (DEFAULT_LIBS "-nodefaultlibs")
|
||||
|
||||
# We need builtins from Clang's RT even without libcxx - for ubsan+int128.
|
||||
# See https://bugs.llvm.org/show_bug.cgi?id=16404
|
||||
# We need builtins from Clang
|
||||
execute_process (COMMAND
|
||||
${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt
|
||||
OUTPUT_VARIABLE BUILTINS_LIBRARY
|
||||
|
||||
4
contrib/CMakeLists.txt
vendored
4
contrib/CMakeLists.txt
vendored
@ -217,7 +217,9 @@ add_contrib (libssh-cmake libssh)
|
||||
|
||||
add_contrib (prometheus-protobufs-cmake prometheus-protobufs prometheus-protobufs-gogo)
|
||||
|
||||
add_contrib(numactl-cmake numactl)
|
||||
add_contrib (numactl-cmake numactl)
|
||||
|
||||
add_contrib (jwt-cpp-cmake jwt-cpp)
|
||||
|
||||
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
|
||||
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
|
||||
|
||||
2
contrib/SimSIMD
vendored
2
contrib/SimSIMD
vendored
@ -1 +1 @@
|
||||
Subproject commit ff51434d90c66f916e94ff05b24530b127aa4cff
|
||||
Subproject commit fa60f1b8e3582c50978f0ae86c2ebb6c9af957f3
|
||||
@ -1,4 +1,8 @@
|
||||
set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD")
|
||||
|
||||
add_library(_simsimd INTERFACE)
|
||||
target_include_directories(_simsimd SYSTEM INTERFACE "${SIMSIMD_PROJECT_DIR}/include")
|
||||
# See contrib/usearch-cmake/CMakeLists.txt, why only enabled on x86
|
||||
if (ARCH_AMD64)
|
||||
set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD")
|
||||
set(SIMSIMD_SRCS ${SIMSIMD_PROJECT_DIR}/c/lib.c)
|
||||
add_library(_simsimd ${SIMSIMD_SRCS})
|
||||
target_include_directories(_simsimd SYSTEM PUBLIC "${SIMSIMD_PROJECT_DIR}/include")
|
||||
target_compile_definitions(_simsimd PUBLIC SIMSIMD_DYNAMIC_DISPATCH)
|
||||
endif()
|
||||
|
||||
2
contrib/arrow
vendored
2
contrib/arrow
vendored
@ -1 +1 @@
|
||||
Subproject commit 5cfccd8ea65f33d4517e7409815d761c7650b45d
|
||||
Subproject commit 6e2574f5013a005c050c9a7787d341aef09d0063
|
||||
@ -213,13 +213,19 @@ target_include_directories(_orc SYSTEM PRIVATE
|
||||
set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src/arrow")
|
||||
|
||||
# arrow/cpp/src/arrow/CMakeLists.txt (ARROW_SRCS + ARROW_COMPUTE + ARROW_IPC)
|
||||
# find . \( -iname \*.cc -o -iname \*.cpp -o -iname \*.c \) | sort | awk '{print "\"${LIBRARY_DIR}" substr($1,2) "\"" }' | grep -v 'test.cc' | grep -v 'json' | grep -v 'flight' \|
|
||||
# grep -v 'csv' | grep -v 'acero' | grep -v 'dataset' | grep -v 'testing' | grep -v 'gpu' | grep -v 'engine' | grep -v 'filesystem' | grep -v 'benchmark.cc'
|
||||
set(ARROW_SRCS
|
||||
"${LIBRARY_DIR}/adapters/orc/adapter.cc"
|
||||
"${LIBRARY_DIR}/adapters/orc/options.cc"
|
||||
"${LIBRARY_DIR}/adapters/orc/util.cc"
|
||||
"${LIBRARY_DIR}/array/array_base.cc"
|
||||
"${LIBRARY_DIR}/array/array_binary.cc"
|
||||
"${LIBRARY_DIR}/array/array_decimal.cc"
|
||||
"${LIBRARY_DIR}/array/array_dict.cc"
|
||||
"${LIBRARY_DIR}/array/array_nested.cc"
|
||||
"${LIBRARY_DIR}/array/array_primitive.cc"
|
||||
"${LIBRARY_DIR}/array/array_run_end.cc"
|
||||
"${LIBRARY_DIR}/array/builder_adaptive.cc"
|
||||
"${LIBRARY_DIR}/array/builder_base.cc"
|
||||
"${LIBRARY_DIR}/array/builder_binary.cc"
|
||||
@ -227,124 +233,26 @@ set(ARROW_SRCS
|
||||
"${LIBRARY_DIR}/array/builder_dict.cc"
|
||||
"${LIBRARY_DIR}/array/builder_nested.cc"
|
||||
"${LIBRARY_DIR}/array/builder_primitive.cc"
|
||||
"${LIBRARY_DIR}/array/builder_union.cc"
|
||||
"${LIBRARY_DIR}/array/builder_run_end.cc"
|
||||
"${LIBRARY_DIR}/array/array_run_end.cc"
|
||||
"${LIBRARY_DIR}/array/builder_union.cc"
|
||||
"${LIBRARY_DIR}/array/concatenate.cc"
|
||||
"${LIBRARY_DIR}/array/data.cc"
|
||||
"${LIBRARY_DIR}/array/diff.cc"
|
||||
"${LIBRARY_DIR}/array/util.cc"
|
||||
"${LIBRARY_DIR}/array/validate.cc"
|
||||
"${LIBRARY_DIR}/builder.cc"
|
||||
"${LIBRARY_DIR}/buffer.cc"
|
||||
"${LIBRARY_DIR}/chunked_array.cc"
|
||||
"${LIBRARY_DIR}/chunk_resolver.cc"
|
||||
"${LIBRARY_DIR}/compare.cc"
|
||||
"${LIBRARY_DIR}/config.cc"
|
||||
"${LIBRARY_DIR}/datum.cc"
|
||||
"${LIBRARY_DIR}/device.cc"
|
||||
"${LIBRARY_DIR}/extension_type.cc"
|
||||
"${LIBRARY_DIR}/memory_pool.cc"
|
||||
"${LIBRARY_DIR}/pretty_print.cc"
|
||||
"${LIBRARY_DIR}/record_batch.cc"
|
||||
"${LIBRARY_DIR}/result.cc"
|
||||
"${LIBRARY_DIR}/scalar.cc"
|
||||
"${LIBRARY_DIR}/sparse_tensor.cc"
|
||||
"${LIBRARY_DIR}/status.cc"
|
||||
"${LIBRARY_DIR}/table.cc"
|
||||
"${LIBRARY_DIR}/table_builder.cc"
|
||||
"${LIBRARY_DIR}/tensor.cc"
|
||||
"${LIBRARY_DIR}/tensor/coo_converter.cc"
|
||||
"${LIBRARY_DIR}/tensor/csf_converter.cc"
|
||||
"${LIBRARY_DIR}/tensor/csx_converter.cc"
|
||||
"${LIBRARY_DIR}/type.cc"
|
||||
"${LIBRARY_DIR}/visitor.cc"
|
||||
"${LIBRARY_DIR}/builder.cc"
|
||||
"${LIBRARY_DIR}/c/bridge.cc"
|
||||
"${LIBRARY_DIR}/io/buffered.cc"
|
||||
"${LIBRARY_DIR}/io/caching.cc"
|
||||
"${LIBRARY_DIR}/io/compressed.cc"
|
||||
"${LIBRARY_DIR}/io/file.cc"
|
||||
"${LIBRARY_DIR}/io/hdfs.cc"
|
||||
"${LIBRARY_DIR}/io/hdfs_internal.cc"
|
||||
"${LIBRARY_DIR}/io/interfaces.cc"
|
||||
"${LIBRARY_DIR}/io/memory.cc"
|
||||
"${LIBRARY_DIR}/io/slow.cc"
|
||||
"${LIBRARY_DIR}/io/stdio.cc"
|
||||
"${LIBRARY_DIR}/io/transform.cc"
|
||||
"${LIBRARY_DIR}/util/async_util.cc"
|
||||
"${LIBRARY_DIR}/util/basic_decimal.cc"
|
||||
"${LIBRARY_DIR}/util/bit_block_counter.cc"
|
||||
"${LIBRARY_DIR}/util/bit_run_reader.cc"
|
||||
"${LIBRARY_DIR}/util/bit_util.cc"
|
||||
"${LIBRARY_DIR}/util/bitmap.cc"
|
||||
"${LIBRARY_DIR}/util/bitmap_builders.cc"
|
||||
"${LIBRARY_DIR}/util/bitmap_ops.cc"
|
||||
"${LIBRARY_DIR}/util/bpacking.cc"
|
||||
"${LIBRARY_DIR}/util/cancel.cc"
|
||||
"${LIBRARY_DIR}/util/compression.cc"
|
||||
"${LIBRARY_DIR}/util/counting_semaphore.cc"
|
||||
"${LIBRARY_DIR}/util/cpu_info.cc"
|
||||
"${LIBRARY_DIR}/util/decimal.cc"
|
||||
"${LIBRARY_DIR}/util/delimiting.cc"
|
||||
"${LIBRARY_DIR}/util/formatting.cc"
|
||||
"${LIBRARY_DIR}/util/future.cc"
|
||||
"${LIBRARY_DIR}/util/int_util.cc"
|
||||
"${LIBRARY_DIR}/util/io_util.cc"
|
||||
"${LIBRARY_DIR}/util/logging.cc"
|
||||
"${LIBRARY_DIR}/util/key_value_metadata.cc"
|
||||
"${LIBRARY_DIR}/util/memory.cc"
|
||||
"${LIBRARY_DIR}/util/mutex.cc"
|
||||
"${LIBRARY_DIR}/util/string.cc"
|
||||
"${LIBRARY_DIR}/util/string_builder.cc"
|
||||
"${LIBRARY_DIR}/util/task_group.cc"
|
||||
"${LIBRARY_DIR}/util/tdigest.cc"
|
||||
"${LIBRARY_DIR}/util/thread_pool.cc"
|
||||
"${LIBRARY_DIR}/util/time.cc"
|
||||
"${LIBRARY_DIR}/util/trie.cc"
|
||||
"${LIBRARY_DIR}/util/unreachable.cc"
|
||||
"${LIBRARY_DIR}/util/uri.cc"
|
||||
"${LIBRARY_DIR}/util/utf8.cc"
|
||||
"${LIBRARY_DIR}/util/value_parsing.cc"
|
||||
"${LIBRARY_DIR}/util/byte_size.cc"
|
||||
"${LIBRARY_DIR}/util/debug.cc"
|
||||
"${LIBRARY_DIR}/util/tracing.cc"
|
||||
"${LIBRARY_DIR}/util/atfork_internal.cc"
|
||||
"${LIBRARY_DIR}/util/crc32.cc"
|
||||
"${LIBRARY_DIR}/util/hashing.cc"
|
||||
"${LIBRARY_DIR}/util/ree_util.cc"
|
||||
"${LIBRARY_DIR}/util/union_util.cc"
|
||||
"${LIBRARY_DIR}/vendored/base64.cpp"
|
||||
"${LIBRARY_DIR}/vendored/datetime/tz.cpp"
|
||||
"${LIBRARY_DIR}/vendored/musl/strptime.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriCommon.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriCompare.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriEscape.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriFile.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriIp4Base.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriIp4.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriMemory.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriNormalizeBase.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriNormalize.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriParseBase.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriParse.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriQuery.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriRecompose.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriResolve.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriShorten.c"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/bignum.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/bignum-dtoa.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/cached-powers.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/double-to-string.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/fast-dtoa.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/fixed-dtoa.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/string-to-double.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/strtod.cc"
|
||||
|
||||
"${LIBRARY_DIR}/c/dlpack.cc"
|
||||
"${LIBRARY_DIR}/chunk_resolver.cc"
|
||||
"${LIBRARY_DIR}/chunked_array.cc"
|
||||
"${LIBRARY_DIR}/compare.cc"
|
||||
"${LIBRARY_DIR}/compute/api_aggregate.cc"
|
||||
"${LIBRARY_DIR}/compute/api_scalar.cc"
|
||||
"${LIBRARY_DIR}/compute/api_vector.cc"
|
||||
"${LIBRARY_DIR}/compute/cast.cc"
|
||||
"${LIBRARY_DIR}/compute/exec.cc"
|
||||
"${LIBRARY_DIR}/compute/expression.cc"
|
||||
"${LIBRARY_DIR}/compute/function.cc"
|
||||
"${LIBRARY_DIR}/compute/function_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/kernel.cc"
|
||||
@ -355,6 +263,7 @@ set(ARROW_SRCS
|
||||
"${LIBRARY_DIR}/compute/kernels/aggregate_var_std.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/codegen_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/hash_aggregate.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/ree_util_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/row_encoder.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/scalar_arithmetic.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/scalar_boolean.cc"
|
||||
@ -382,30 +291,139 @@ set(ARROW_SRCS
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_cumulative_ops.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_hash.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_nested.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_pairwise.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_rank.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_replace.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_run_end_encode.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_select_k.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_selection.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_sort.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_selection_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_selection_filter_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_selection_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_selection_take_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/light_array.cc"
|
||||
"${LIBRARY_DIR}/compute/registry.cc"
|
||||
"${LIBRARY_DIR}/compute/expression.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/vector_sort.cc"
|
||||
"${LIBRARY_DIR}/compute/key_hash_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/key_map_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/light_array_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/ordering.cc"
|
||||
"${LIBRARY_DIR}/compute/registry.cc"
|
||||
"${LIBRARY_DIR}/compute/row/compare_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/row/encode_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/row/grouper.cc"
|
||||
"${LIBRARY_DIR}/compute/row/row_internal.cc"
|
||||
|
||||
"${LIBRARY_DIR}/compute/util.cc"
|
||||
"${LIBRARY_DIR}/config.cc"
|
||||
"${LIBRARY_DIR}/datum.cc"
|
||||
"${LIBRARY_DIR}/device.cc"
|
||||
"${LIBRARY_DIR}/extension_type.cc"
|
||||
"${LIBRARY_DIR}/integration/c_data_integration_internal.cc"
|
||||
"${LIBRARY_DIR}/io/buffered.cc"
|
||||
"${LIBRARY_DIR}/io/caching.cc"
|
||||
"${LIBRARY_DIR}/io/compressed.cc"
|
||||
"${LIBRARY_DIR}/io/file.cc"
|
||||
"${LIBRARY_DIR}/io/hdfs.cc"
|
||||
"${LIBRARY_DIR}/io/hdfs_internal.cc"
|
||||
"${LIBRARY_DIR}/io/interfaces.cc"
|
||||
"${LIBRARY_DIR}/io/memory.cc"
|
||||
"${LIBRARY_DIR}/io/slow.cc"
|
||||
"${LIBRARY_DIR}/io/stdio.cc"
|
||||
"${LIBRARY_DIR}/io/transform.cc"
|
||||
"${LIBRARY_DIR}/ipc/dictionary.cc"
|
||||
"${LIBRARY_DIR}/ipc/feather.cc"
|
||||
"${LIBRARY_DIR}/ipc/file_to_stream.cc"
|
||||
"${LIBRARY_DIR}/ipc/message.cc"
|
||||
"${LIBRARY_DIR}/ipc/metadata_internal.cc"
|
||||
"${LIBRARY_DIR}/ipc/options.cc"
|
||||
"${LIBRARY_DIR}/ipc/reader.cc"
|
||||
"${LIBRARY_DIR}/ipc/stream_to_file.cc"
|
||||
"${LIBRARY_DIR}/ipc/writer.cc"
|
||||
"${LIBRARY_DIR}/memory_pool.cc"
|
||||
"${LIBRARY_DIR}/pretty_print.cc"
|
||||
"${LIBRARY_DIR}/record_batch.cc"
|
||||
"${LIBRARY_DIR}/result.cc"
|
||||
"${LIBRARY_DIR}/scalar.cc"
|
||||
"${LIBRARY_DIR}/sparse_tensor.cc"
|
||||
"${LIBRARY_DIR}/status.cc"
|
||||
"${LIBRARY_DIR}/table.cc"
|
||||
"${LIBRARY_DIR}/table_builder.cc"
|
||||
"${LIBRARY_DIR}/tensor.cc"
|
||||
"${LIBRARY_DIR}/tensor/coo_converter.cc"
|
||||
"${LIBRARY_DIR}/tensor/csf_converter.cc"
|
||||
"${LIBRARY_DIR}/tensor/csx_converter.cc"
|
||||
"${LIBRARY_DIR}/type.cc"
|
||||
"${LIBRARY_DIR}/type_traits.cc"
|
||||
"${LIBRARY_DIR}/util/align_util.cc"
|
||||
"${LIBRARY_DIR}/util/async_util.cc"
|
||||
"${LIBRARY_DIR}/util/atfork_internal.cc"
|
||||
"${LIBRARY_DIR}/util/basic_decimal.cc"
|
||||
"${LIBRARY_DIR}/util/bit_block_counter.cc"
|
||||
"${LIBRARY_DIR}/util/bit_run_reader.cc"
|
||||
"${LIBRARY_DIR}/util/bit_util.cc"
|
||||
"${LIBRARY_DIR}/util/bitmap.cc"
|
||||
"${LIBRARY_DIR}/util/bitmap_builders.cc"
|
||||
"${LIBRARY_DIR}/util/bitmap_ops.cc"
|
||||
"${LIBRARY_DIR}/util/bpacking.cc"
|
||||
"${LIBRARY_DIR}/util/byte_size.cc"
|
||||
"${LIBRARY_DIR}/util/cancel.cc"
|
||||
"${LIBRARY_DIR}/util/compression.cc"
|
||||
"${LIBRARY_DIR}/util/counting_semaphore.cc"
|
||||
"${LIBRARY_DIR}/util/cpu_info.cc"
|
||||
"${LIBRARY_DIR}/util/crc32.cc"
|
||||
"${LIBRARY_DIR}/util/debug.cc"
|
||||
"${LIBRARY_DIR}/util/decimal.cc"
|
||||
"${LIBRARY_DIR}/util/delimiting.cc"
|
||||
"${LIBRARY_DIR}/util/dict_util.cc"
|
||||
"${LIBRARY_DIR}/util/float16.cc"
|
||||
"${LIBRARY_DIR}/util/formatting.cc"
|
||||
"${LIBRARY_DIR}/util/future.cc"
|
||||
"${LIBRARY_DIR}/util/hashing.cc"
|
||||
"${LIBRARY_DIR}/util/int_util.cc"
|
||||
"${LIBRARY_DIR}/util/io_util.cc"
|
||||
"${LIBRARY_DIR}/util/key_value_metadata.cc"
|
||||
"${LIBRARY_DIR}/util/list_util.cc"
|
||||
"${LIBRARY_DIR}/util/logging.cc"
|
||||
"${LIBRARY_DIR}/util/memory.cc"
|
||||
"${LIBRARY_DIR}/util/mutex.cc"
|
||||
"${LIBRARY_DIR}/util/ree_util.cc"
|
||||
"${LIBRARY_DIR}/util/string.cc"
|
||||
"${LIBRARY_DIR}/util/string_builder.cc"
|
||||
"${LIBRARY_DIR}/util/task_group.cc"
|
||||
"${LIBRARY_DIR}/util/tdigest.cc"
|
||||
"${LIBRARY_DIR}/util/thread_pool.cc"
|
||||
"${LIBRARY_DIR}/util/time.cc"
|
||||
"${LIBRARY_DIR}/util/tracing.cc"
|
||||
"${LIBRARY_DIR}/util/trie.cc"
|
||||
"${LIBRARY_DIR}/util/union_util.cc"
|
||||
"${LIBRARY_DIR}/util/unreachable.cc"
|
||||
"${LIBRARY_DIR}/util/uri.cc"
|
||||
"${LIBRARY_DIR}/util/utf8.cc"
|
||||
"${LIBRARY_DIR}/util/value_parsing.cc"
|
||||
"${LIBRARY_DIR}/vendored/base64.cpp"
|
||||
"${LIBRARY_DIR}/vendored/datetime/tz.cpp"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/bignum-dtoa.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/bignum.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/cached-powers.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/double-to-string.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/fast-dtoa.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/fixed-dtoa.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/string-to-double.cc"
|
||||
"${LIBRARY_DIR}/vendored/double-conversion/strtod.cc"
|
||||
"${LIBRARY_DIR}/vendored/musl/strptime.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriCommon.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriCompare.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriEscape.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriFile.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriIp4.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriIp4Base.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriMemory.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriNormalize.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriNormalizeBase.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriParse.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriParseBase.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriQuery.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriRecompose.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriResolve.c"
|
||||
"${LIBRARY_DIR}/vendored/uriparser/UriShorten.c"
|
||||
"${LIBRARY_DIR}/visitor.cc"
|
||||
|
||||
"${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc"
|
||||
"${ARROW_SRC_DIR}/arrow/adapters/orc/util.cc"
|
||||
@ -465,22 +483,38 @@ set(PARQUET_SRCS
|
||||
"${LIBRARY_DIR}/arrow/schema.cc"
|
||||
"${LIBRARY_DIR}/arrow/schema_internal.cc"
|
||||
"${LIBRARY_DIR}/arrow/writer.cc"
|
||||
"${LIBRARY_DIR}/benchmark_util.cc"
|
||||
"${LIBRARY_DIR}/bloom_filter.cc"
|
||||
"${LIBRARY_DIR}/bloom_filter_reader.cc"
|
||||
"${LIBRARY_DIR}/column_reader.cc"
|
||||
"${LIBRARY_DIR}/column_scanner.cc"
|
||||
"${LIBRARY_DIR}/column_writer.cc"
|
||||
"${LIBRARY_DIR}/encoding.cc"
|
||||
"${LIBRARY_DIR}/encryption/crypto_factory.cc"
|
||||
"${LIBRARY_DIR}/encryption/encryption.cc"
|
||||
"${LIBRARY_DIR}/encryption/encryption_internal.cc"
|
||||
"${LIBRARY_DIR}/encryption/encryption_internal_nossl.cc"
|
||||
"${LIBRARY_DIR}/encryption/file_key_unwrapper.cc"
|
||||
"${LIBRARY_DIR}/encryption/file_key_wrapper.cc"
|
||||
"${LIBRARY_DIR}/encryption/file_system_key_material_store.cc"
|
||||
"${LIBRARY_DIR}/encryption/internal_file_decryptor.cc"
|
||||
"${LIBRARY_DIR}/encryption/internal_file_encryptor.cc"
|
||||
"${LIBRARY_DIR}/encryption/key_material.cc"
|
||||
"${LIBRARY_DIR}/encryption/key_metadata.cc"
|
||||
"${LIBRARY_DIR}/encryption/key_toolkit.cc"
|
||||
"${LIBRARY_DIR}/encryption/key_toolkit_internal.cc"
|
||||
"${LIBRARY_DIR}/encryption/kms_client.cc"
|
||||
"${LIBRARY_DIR}/encryption/local_wrap_kms_client.cc"
|
||||
"${LIBRARY_DIR}/encryption/openssl_internal.cc"
|
||||
"${LIBRARY_DIR}/exception.cc"
|
||||
"${LIBRARY_DIR}/file_reader.cc"
|
||||
"${LIBRARY_DIR}/file_writer.cc"
|
||||
"${LIBRARY_DIR}/page_index.cc"
|
||||
"${LIBRARY_DIR}/level_conversion.cc"
|
||||
"${LIBRARY_DIR}/level_comparison.cc"
|
||||
"${LIBRARY_DIR}/level_comparison_avx2.cc"
|
||||
"${LIBRARY_DIR}/level_conversion.cc"
|
||||
"${LIBRARY_DIR}/level_conversion_bmi2.cc"
|
||||
"${LIBRARY_DIR}/metadata.cc"
|
||||
"${LIBRARY_DIR}/page_index.cc"
|
||||
"${LIBRARY_DIR}/platform.cc"
|
||||
"${LIBRARY_DIR}/printer.cc"
|
||||
"${LIBRARY_DIR}/properties.cc"
|
||||
@ -489,7 +523,6 @@ set(PARQUET_SRCS
|
||||
"${LIBRARY_DIR}/stream_reader.cc"
|
||||
"${LIBRARY_DIR}/stream_writer.cc"
|
||||
"${LIBRARY_DIR}/types.cc"
|
||||
"${LIBRARY_DIR}/bloom_filter_reader.cc"
|
||||
"${LIBRARY_DIR}/xxhasher.cc"
|
||||
|
||||
"${GEN_LIBRARY_DIR}/parquet_constants.cpp"
|
||||
@ -520,6 +553,9 @@ endif ()
|
||||
add_definitions(-DPARQUET_THRIFT_VERSION_MAJOR=0)
|
||||
add_definitions(-DPARQUET_THRIFT_VERSION_MINOR=16)
|
||||
|
||||
# As per https://github.com/apache/arrow/pull/35672 you need to enable it explicitly.
|
||||
add_definitions(-DARROW_ENABLE_THREADING)
|
||||
|
||||
# === tools
|
||||
|
||||
set(TOOLS_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/tools/parquet")
|
||||
|
||||
2
contrib/flatbuffers
vendored
2
contrib/flatbuffers
vendored
@ -1 +1 @@
|
||||
Subproject commit eb3f827948241ce0e701516f16cd67324802bce9
|
||||
Subproject commit 0100f6a5779831fa7a651e4b67ef389a8752bd9b
|
||||
1
contrib/jwt-cpp
vendored
Submodule
1
contrib/jwt-cpp
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit a6927cb8140858c34e05d1a954626b9849fbcdfc
|
||||
23
contrib/jwt-cpp-cmake/CMakeLists.txt
Normal file
23
contrib/jwt-cpp-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,23 @@
|
||||
set(ENABLE_JWT_CPP_DEFAULT OFF)
|
||||
if(ENABLE_LIBRARIES AND CLICKHOUSE_CLOUD)
|
||||
set(ENABLE_JWT_CPP_DEFAULT ON)
|
||||
endif()
|
||||
|
||||
option(ENABLE_JWT_CPP "Enable jwt-cpp library" ${ENABLE_JWT_CPP_DEFAULT})
|
||||
|
||||
if (NOT ENABLE_JWT_CPP)
|
||||
message(STATUS "Not using jwt-cpp")
|
||||
return()
|
||||
endif()
|
||||
|
||||
if(ENABLE_JWT_CPP)
|
||||
if(NOT TARGET OpenSSL::Crypto)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use jwt-cpp without OpenSSL")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set (JWT_CPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/jwt-cpp/include")
|
||||
|
||||
add_library (_jwt-cpp INTERFACE)
|
||||
target_include_directories(_jwt-cpp SYSTEM BEFORE INTERFACE ${JWT_CPP_INCLUDE_DIR})
|
||||
add_library(ch_contrib::jwt-cpp ALIAS _jwt-cpp)
|
||||
2
contrib/krb5
vendored
2
contrib/krb5
vendored
@ -1 +1 @@
|
||||
Subproject commit 71b06c2276009ae649c7703019f3b4605f66fd3d
|
||||
Subproject commit c5b4b994c18db86933255907a97eee5993fd18fe
|
||||
2
contrib/usearch
vendored
2
contrib/usearch
vendored
@ -1 +1 @@
|
||||
Subproject commit 1706420acafbd83d852c512dcf343af0a4059e48
|
||||
Subproject commit 7efe8b710c9831bfe06573b1df0fad001b04a2b5
|
||||
@ -6,12 +6,63 @@ target_include_directories(_usearch SYSTEM INTERFACE ${USEARCH_PROJECT_DIR}/incl
|
||||
target_link_libraries(_usearch INTERFACE _fp16)
|
||||
target_compile_definitions(_usearch INTERFACE USEARCH_USE_FP16LIB)
|
||||
|
||||
# target_compile_definitions(_usearch INTERFACE USEARCH_USE_SIMSIMD)
|
||||
# ^^ simsimd is not enabled at the moment. Reasons:
|
||||
# - Vectorization is important for raw scans but not so much for HNSW. We use usearch only for HNSW.
|
||||
# - Simsimd does compile-time dispatch (choice of SIMD kernels determined by capabilities of the build machine) or dynamic dispatch (SIMD
|
||||
# kernels chosen at runtime based on cpuid instruction). Since current builds are limited to SSE 4.2 (x86) and NEON (ARM), the speedup of
|
||||
# the former would be moderate compared to AVX-512 / SVE. The latter is at the moment too fragile with respect to portability across x86
|
||||
# and ARM machines ... certain conbinations of quantizations / distance functions / SIMD instructions are not implemented at the moment.
|
||||
# Only x86 for now. On ARM, the linker goes down in flames. To make SimSIMD compile, I had to remove a macro checks in SimSIMD
|
||||
# for AVX512 (x86, worked nicely) and __ARM_BF16_FORMAT_ALTERNATIVE. It is probably because of that.
|
||||
if (ARCH_AMD64)
|
||||
target_link_libraries(_usearch INTERFACE _simsimd)
|
||||
target_compile_definitions(_usearch INTERFACE USEARCH_USE_SIMSIMD)
|
||||
|
||||
target_compile_definitions(_usearch INTERFACE USEARCH_CAN_COMPILE_FLOAT16)
|
||||
target_compile_definitions(_usearch INTERFACE USEARCH_CAN_COMPILE_BF16)
|
||||
endif ()
|
||||
|
||||
add_library(ch_contrib::usearch ALIAS _usearch)
|
||||
|
||||
|
||||
# Cf. https://github.com/llvm/llvm-project/issues/107810 (though it is not 100% the same stack)
|
||||
#
|
||||
# LLVM ERROR: Cannot select: 0x7996e7a73150: f32,ch = load<(load (s16) from %ir.22, !tbaa !54231), anyext from bf16> 0x79961cb737c0, 0x7996e7a1a500, undef:i64, ./contrib/SimSIMD/include/simsimd/dot.h:215:1
|
||||
# 0x7996e7a1a500: i64 = add 0x79961e770d00, Constant:i64<-16>, ./contrib/SimSIMD/include/simsimd/dot.h:215:1
|
||||
# 0x79961e770d00: i64,ch = CopyFromReg 0x79961cb737c0, Register:i64 %4, ./contrib/SimSIMD/include/simsimd/dot.h:215:1
|
||||
# 0x7996e7a1ae10: i64 = Register %4
|
||||
# 0x7996e7a1b5f0: i64 = Constant<-16>
|
||||
# 0x7996e7a1a730: i64 = undef
|
||||
# In function: _ZL23simsimd_dot_bf16_serialPKu6__bf16S0_yPd
|
||||
# PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
|
||||
# Stack dump:
|
||||
# 0. Running pass 'Function Pass Manager' on module 'src/libdbms.a(MergeTreeIndexVectorSimilarity.cpp.o at 2312737440)'.
|
||||
# 1. Running pass 'AArch64 Instruction Selection' on function '@_ZL23simsimd_dot_bf16_serialPKu6__bf16S0_yPd'
|
||||
# #0 0x00007999e83a63bf llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xda63bf)
|
||||
# #1 0x00007999e83a44f9 llvm::sys::RunSignalHandlers() (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xda44f9)
|
||||
# #2 0x00007999e83a6b00 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xda6b00)
|
||||
# #3 0x00007999e6e45320 (/lib/x86_64-linux-gnu/libc.so.6+0x45320)
|
||||
# #4 0x00007999e6e9eb1c pthread_kill (/lib/x86_64-linux-gnu/libc.so.6+0x9eb1c)
|
||||
# #5 0x00007999e6e4526e raise (/lib/x86_64-linux-gnu/libc.so.6+0x4526e)
|
||||
# #6 0x00007999e6e288ff abort (/lib/x86_64-linux-gnu/libc.so.6+0x288ff)
|
||||
# #7 0x00007999e82fe0c2 llvm::report_fatal_error(llvm::Twine const&, bool) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xcfe0c2)
|
||||
# #8 0x00007999e8c2f8e3 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x162f8e3)
|
||||
# #9 0x00007999e8c2ed76 llvm::SelectionDAGISel::SelectCodeCommon(llvm::SDNode*, unsigned char const*, unsigned int) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x162ed76)
|
||||
# #10 0x00007999ea1adbcb (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x2badbcb)
|
||||
# #11 0x00007999e8c2611f llvm::SelectionDAGISel::DoInstructionSelection() (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x162611f)
|
||||
# #12 0x00007999e8c25790 llvm::SelectionDAGISel::CodeGenAndEmitDAG() (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x1625790)
|
||||
# #13 0x00007999e8c248de llvm::SelectionDAGISel::SelectAllBasicBlocks(llvm::Function const&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x16248de)
|
||||
# #14 0x00007999e8c22934 llvm::SelectionDAGISel::runOnMachineFunction(llvm::MachineFunction&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x1622934)
|
||||
# #15 0x00007999e87826b9 llvm::MachineFunctionPass::runOnFunction(llvm::Function&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x11826b9)
|
||||
# #16 0x00007999e84f7772 llvm::FPPassManager::runOnFunction(llvm::Function&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xef7772)
|
||||
# #17 0x00007999e84fd2f4 llvm::FPPassManager::runOnModule(llvm::Module&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xefd2f4)
|
||||
# #18 0x00007999e84f7e9f llvm::legacy::PassManagerImpl::run(llvm::Module&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xef7e9f)
|
||||
# #19 0x00007999e99f7d61 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f7d61)
|
||||
# #20 0x00007999e99f8c91 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f8c91)
|
||||
# #21 0x00007999e99f8b10 llvm::lto::thinBackend(llvm::lto::Config const&, unsigned int, std::function<llvm::Expected<std::unique_ptr<llvm::CachedFileStream, std::default_delete<llvm::CachedFileStream>>> (unsigned int, llvm::Twine const&)>, llvm::Module&, llvm::ModuleSummaryIndex const&, llvm::DenseMap<llvm::StringRef, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long>>, llvm::DenseMapInfo<llvm::StringRef, void
|
||||
# >, llvm::detail::DenseMapPair<llvm::StringRef, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long>>>> const&, llvm::DenseMap<unsigned long, llvm::GlobalValueSummary*, llvm::DenseMapInfo<unsigned long, void>, llvm::detail::DenseMapPair<unsigned long, llvm::GlobalValueSummary*>> const&, llvm::MapVector<llvm::StringRef, llvm::BitcodeModule, llvm::DenseMap<llvm::StringRef, unsigned int, llvm::DenseMapInfo<llvm::S
|
||||
# tringRef, void>, llvm::detail::DenseMapPair<llvm::StringRef, unsigned int>>, llvm::SmallVector<std::pair<llvm::StringRef, llvm::BitcodeModule>, 0u>>*, std::vector<unsigned char, std::allocator<unsigned char>> const&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f8b10)
|
||||
# #22 0x00007999e99f248d (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f248d)
|
||||
# #23 0x00007999e99f1cd6 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f1cd6)
|
||||
# #24 0x00007999e82c9beb (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xcc9beb)
|
||||
# #25 0x00007999e834ebe3 llvm::ThreadPool::processTasks(llvm::ThreadPoolTaskGroup*) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xd4ebe3)
|
||||
# #26 0x00007999e834f704 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xd4f704)
|
||||
# #27 0x00007999e6e9ca94 (/lib/x86_64-linux-gnu/libc.so.6+0x9ca94)
|
||||
# #28 0x00007999e6f29c3c (/lib/x86_64-linux-gnu/libc.so.6+0x129c3c)
|
||||
# clang++-18: error: unable to execute command: Aborted (core dumped)
|
||||
# clang++-18: error: linker command failed due to signal (use -v to see invocation)
|
||||
# ^[[A^Cninja: build stopped: interrupted by user.
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# The Dockerfile.ubuntu exists for the tests/ci/docker_server.py script
|
||||
# If the image is built from Dockerfile.alpine, then the `-alpine` suffix is added automatically,
|
||||
# so the only purpose of Dockerfile.ubuntu is to push `latest`, `head` and so on w/o suffixes
|
||||
FROM ubuntu:20.04 AS glibc-donor
|
||||
FROM ubuntu:22.04 AS glibc-donor
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN arch=${TARGETARCH:-amd64} \
|
||||
@ -9,7 +9,11 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
amd64) rarch=x86_64 ;; \
|
||||
arm64) rarch=aarch64 ;; \
|
||||
esac \
|
||||
&& ln -s "${rarch}-linux-gnu" /lib/linux-gnu
|
||||
&& ln -s "${rarch}-linux-gnu" /lib/linux-gnu \
|
||||
&& case $arch in \
|
||||
amd64) ln /lib/linux-gnu/ld-linux-x86-64.so.2 /lib/linux-gnu/ld-2.35.so ;; \
|
||||
arm64) ln /lib/linux-gnu/ld-linux-aarch64.so.1 /lib/linux-gnu/ld-2.35.so ;; \
|
||||
esac
|
||||
|
||||
|
||||
FROM alpine
|
||||
@ -20,21 +24,21 @@ ENV LANG=en_US.UTF-8 \
|
||||
TZ=UTC \
|
||||
CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml
|
||||
|
||||
COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.31.so /lib/
|
||||
COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.35.so /lib/
|
||||
COPY --from=glibc-donor /etc/nsswitch.conf /etc/
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
ARG TARGETARCH
|
||||
RUN arch=${TARGETARCH:-amd64} \
|
||||
&& case $arch in \
|
||||
amd64) mkdir -p /lib64 && ln -sf /lib/ld-2.31.so /lib64/ld-linux-x86-64.so.2 ;; \
|
||||
arm64) ln -sf /lib/ld-2.31.so /lib/ld-linux-aarch64.so.1 ;; \
|
||||
amd64) mkdir -p /lib64 && ln -sf /lib/ld-2.35.so /lib64/ld-linux-x86-64.so.2 ;; \
|
||||
arm64) ln -sf /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1 ;; \
|
||||
esac
|
||||
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.9.2.42"
|
||||
ARG VERSION="24.10.2.80"
|
||||
ARG PACKAGES="clickhouse-keeper"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
@ -82,7 +86,8 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
ARG DEFAULT_CONFIG_DIR="/etc/clickhouse-keeper"
|
||||
ARG DEFAULT_DATA_DIR="/var/lib/clickhouse-keeper"
|
||||
ARG DEFAULT_LOG_DIR="/var/log/clickhouse-keeper"
|
||||
RUN mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" \
|
||||
RUN clickhouse-keeper --version \
|
||||
&& mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" \
|
||||
&& chown clickhouse:clickhouse "${DEFAULT_DATA_DIR}" \
|
||||
&& chown root:clickhouse "${DEFAULT_LOG_DIR}" \
|
||||
&& chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}"
|
||||
|
||||
@ -1,21 +1,31 @@
|
||||
#!/bin/bash
|
||||
|
||||
set +x
|
||||
set -eo pipefail
|
||||
shopt -s nullglob
|
||||
|
||||
DO_CHOWN=1
|
||||
if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then
|
||||
if [[ "${CLICKHOUSE_RUN_AS_ROOT:=0}" = "1" || "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]]; then
|
||||
DO_CHOWN=0
|
||||
fi
|
||||
|
||||
CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
|
||||
CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
|
||||
# CLICKHOUSE_UID and CLICKHOUSE_GID are kept for backward compatibility, but deprecated
|
||||
# One must use either "docker run --user" or CLICKHOUSE_RUN_AS_ROOT=1 to run the process as
|
||||
# FIXME: Remove ALL CLICKHOUSE_UID CLICKHOUSE_GID before 25.3
|
||||
if [[ "${CLICKHOUSE_UID:-}" || "${CLICKHOUSE_GID:-}" ]]; then
|
||||
echo 'WARNING: Support for CLICKHOUSE_UID/CLICKHOUSE_GID will be removed in a couple of releases.' >&2
|
||||
echo 'WARNING: Either use a proper "docker run --user=xxx:xxxx" argument instead of CLICKHOUSE_UID/CLICKHOUSE_GID' >&2
|
||||
echo 'WARNING: or set "CLICKHOUSE_RUN_AS_ROOT=1" ENV to run the clickhouse-server as root:root' >&2
|
||||
fi
|
||||
|
||||
# support --user
|
||||
if [ "$(id -u)" = "0" ]; then
|
||||
USER=$CLICKHOUSE_UID
|
||||
GROUP=$CLICKHOUSE_GID
|
||||
# support `docker run --user=xxx:xxxx`
|
||||
if [[ "$(id -u)" = "0" ]]; then
|
||||
if [[ "$CLICKHOUSE_RUN_AS_ROOT" = 1 ]]; then
|
||||
USER=0
|
||||
GROUP=0
|
||||
else
|
||||
USER="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
|
||||
GROUP="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
|
||||
fi
|
||||
if command -v gosu &> /dev/null; then
|
||||
gosu="gosu $USER:$GROUP"
|
||||
elif command -v su-exec &> /dev/null; then
|
||||
@ -82,11 +92,11 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
|
||||
|
||||
# There is a config file. It is already tested with gosu (if it is readably by keeper user)
|
||||
if [ -f "$KEEPER_CONFIG" ]; then
|
||||
exec $gosu /usr/bin/clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@"
|
||||
exec $gosu clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@"
|
||||
fi
|
||||
|
||||
# There is no config file. Will use embedded one
|
||||
exec $gosu /usr/bin/clickhouse-keeper --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@"
|
||||
exec $gosu clickhouse-keeper --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@"
|
||||
fi
|
||||
|
||||
# Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image
|
||||
|
||||
@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.9.2.42"
|
||||
ARG VERSION="24.10.2.80"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
FROM ubuntu:20.04
|
||||
FROM ubuntu:22.04
|
||||
|
||||
# see https://github.com/moby/moby/issues/4032#issuecomment-192327844
|
||||
# It could be removed after we move on a version 23:04+
|
||||
@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
|
||||
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
|
||||
ARG VERSION="24.9.2.42"
|
||||
ARG VERSION="24.10.2.80"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
#docker-official-library:off
|
||||
@ -88,34 +88,34 @@ RUN if [ -n "${single_binary_location_url}" ]; then \
|
||||
#docker-official-library:on
|
||||
|
||||
# A fallback to installation from ClickHouse repository
|
||||
RUN if ! clickhouse local -q "SELECT ''" > /dev/null 2>&1; then \
|
||||
apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
apt-transport-https \
|
||||
dirmngr \
|
||||
gnupg2 \
|
||||
&& mkdir -p /etc/apt/sources.list.d \
|
||||
&& GNUPGHOME=$(mktemp -d) \
|
||||
&& GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \
|
||||
--keyring /usr/share/keyrings/clickhouse-keyring.gpg \
|
||||
--keyserver hkp://keyserver.ubuntu.com:80 \
|
||||
--recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \
|
||||
&& rm -rf "$GNUPGHOME" \
|
||||
&& chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
|
||||
&& echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
|
||||
&& echo "installing from repository: ${REPOSITORY}" \
|
||||
&& apt-get update \
|
||||
&& for package in ${PACKAGES}; do \
|
||||
packages="${packages} ${package}=${VERSION}" \
|
||||
; done \
|
||||
&& apt-get install --allow-unauthenticated --yes --no-install-recommends ${packages} || exit 1 \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
/tmp/* \
|
||||
&& apt-get autoremove --purge -yq libksba8 \
|
||||
&& apt-get autoremove -yq \
|
||||
; fi
|
||||
# It works unless the clickhouse binary already exists
|
||||
RUN clickhouse local -q 'SELECT 1' >/dev/null 2>&1 && exit 0 || : \
|
||||
; apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
dirmngr \
|
||||
gnupg2 \
|
||||
&& mkdir -p /etc/apt/sources.list.d \
|
||||
&& GNUPGHOME=$(mktemp -d) \
|
||||
&& GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \
|
||||
--keyring /usr/share/keyrings/clickhouse-keyring.gpg \
|
||||
--keyserver hkp://keyserver.ubuntu.com:80 \
|
||||
--recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \
|
||||
&& rm -rf "$GNUPGHOME" \
|
||||
&& chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
|
||||
&& echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
|
||||
&& echo "installing from repository: ${REPOSITORY}" \
|
||||
&& apt-get update \
|
||||
&& for package in ${PACKAGES}; do \
|
||||
packages="${packages} ${package}=${VERSION}" \
|
||||
; done \
|
||||
&& apt-get install --yes --no-install-recommends ${packages} || exit 1 \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
/tmp/* \
|
||||
&& apt-get autoremove --purge -yq dirmngr gnupg2 \
|
||||
&& chmod ugo+Xrw -R /etc/clickhouse-server /etc/clickhouse-client
|
||||
# The last chmod is here to make the next one is No-op in docker official library Dockerfile
|
||||
|
||||
# post install
|
||||
# we need to allow "others" access to clickhouse folder, because docker container
|
||||
@ -126,8 +126,6 @@ RUN clickhouse-local -q 'SELECT * FROM system.build_options' \
|
||||
|
||||
RUN locale-gen en_US.UTF-8
|
||||
ENV LANG en_US.UTF-8
|
||||
ENV LANGUAGE en_US:en
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
ENV TZ UTC
|
||||
|
||||
RUN mkdir /docker-entrypoint-initdb.d
|
||||
|
||||
@ -1,3 +1,11 @@
|
||||
<!---
|
||||
The README.md is generated by README.sh from the following sources:
|
||||
- README.src/content.md
|
||||
- README.src/license.md
|
||||
|
||||
If you want to change it, edit these files
|
||||
-->
|
||||
|
||||
# ClickHouse Server Docker Image
|
||||
|
||||
## What is ClickHouse?
|
||||
@ -8,6 +16,7 @@ ClickHouse works 100-1000x faster than traditional database management systems,
|
||||
|
||||
For more information and documentation see https://clickhouse.com/.
|
||||
|
||||
<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
|
||||
## Versions
|
||||
|
||||
- The `latest` tag points to the latest release of the latest stable branch.
|
||||
@ -16,10 +25,12 @@ For more information and documentation see https://clickhouse.com/.
|
||||
- The tag `head` is built from the latest commit to the default branch.
|
||||
- Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
|
||||
|
||||
<!-- REMOVE UNTIL HERE -->
|
||||
### Compatibility
|
||||
|
||||
- The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
|
||||
- The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A).
|
||||
- Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run --security-opt seccomp=unconfined` instead, however that has security implications.
|
||||
|
||||
## How to use this image
|
||||
|
||||
@ -29,7 +40,7 @@ For more information and documentation see https://clickhouse.com/.
|
||||
docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
|
||||
```
|
||||
|
||||
By default, ClickHouse will be accessible only via the Docker network. See the [networking section below](#networking).
|
||||
By default, ClickHouse will be accessible only via the Docker network. See the **networking** section below.
|
||||
|
||||
By default, starting above server instance will be run as the `default` user without password.
|
||||
|
||||
@ -46,7 +57,7 @@ More information about the [ClickHouse client](https://clickhouse.com/docs/en/in
|
||||
### connect to it using curl
|
||||
|
||||
```bash
|
||||
echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server curlimages/curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
|
||||
echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server buildpack-deps:curl curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
|
||||
```
|
||||
|
||||
More information about the [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/).
|
||||
@ -69,7 +80,7 @@ echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @-
|
||||
|
||||
`22.6.3.35`
|
||||
|
||||
or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
|
||||
Or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
|
||||
|
||||
```bash
|
||||
docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
|
||||
@ -87,8 +98,8 @@ Typically you may want to mount the following folders inside your container to a
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
-v $(realpath ./ch_data):/var/lib/clickhouse/ \
|
||||
-v $(realpath ./ch_logs):/var/log/clickhouse-server/ \
|
||||
-v "$PWD/ch_data:/var/lib/clickhouse/" \
|
||||
-v "$PWD/ch_logs:/var/log/clickhouse-server/" \
|
||||
--name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
|
||||
```
|
||||
|
||||
@ -110,6 +121,8 @@ docker run -d \
|
||||
--name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
|
||||
```
|
||||
|
||||
Read more in [knowledge base](https://clickhouse.com/docs/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker).
|
||||
|
||||
## Configuration
|
||||
|
||||
The container exposes port 8123 for the [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and port 9000 for the [native client](https://clickhouse.com/docs/en/interfaces/tcp/).
|
||||
@ -125,8 +138,8 @@ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /pa
|
||||
### Start server as custom user
|
||||
|
||||
```bash
|
||||
# $(pwd)/data/clickhouse should exist and be owned by current user
|
||||
docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
|
||||
# $PWD/data/clickhouse should exist and be owned by current user
|
||||
docker run --rm --user "${UID}:${GID}" --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
|
||||
```
|
||||
|
||||
When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start.
|
||||
@ -134,7 +147,7 @@ When you use the image with local directories mounted, you probably want to spec
|
||||
### Start server from root (useful in case of enabled user namespace)
|
||||
|
||||
```bash
|
||||
docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server-userns -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
|
||||
docker run --rm -e CLICKHOUSE_RUN_AS_ROOT=1 --name clickhouse-server-userns -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
|
||||
```
|
||||
|
||||
### How to create default database and user on starting
|
||||
|
||||
38
docker/server/README.sh
Executable file
38
docker/server/README.sh
Executable file
@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
set -ueo pipefail
|
||||
|
||||
# A script to generate README.sh close to as it done in https://github.com/docker-library/docs
|
||||
|
||||
WORKDIR=$(dirname "$0")
|
||||
SCRIPT_NAME=$(basename "$0")
|
||||
CONTENT=README.src/content.md
|
||||
LICENSE=README.src/license.md
|
||||
cd "$WORKDIR"
|
||||
|
||||
R=README.md
|
||||
|
||||
cat > "$R" <<EOD
|
||||
<!---
|
||||
The $R is generated by $SCRIPT_NAME from the following sources:
|
||||
- $CONTENT
|
||||
- $LICENSE
|
||||
|
||||
If you want to change it, edit these files
|
||||
-->
|
||||
|
||||
EOD
|
||||
|
||||
cat "$CONTENT" >> "$R"
|
||||
|
||||
cat >> "$R" <<EOD
|
||||
|
||||
## License
|
||||
|
||||
$(cat $LICENSE)
|
||||
EOD
|
||||
|
||||
# Remove %%LOGO%% from the file with one line below
|
||||
sed -i '/^%%LOGO%%/,+1d' "$R"
|
||||
|
||||
# Replace each %%IMAGE%% with our `clickhouse/clickhouse-server`
|
||||
sed -i '/%%IMAGE%%/s:%%IMAGE%%:clickhouse/clickhouse-server:g' $R
|
||||
1
docker/server/README.src/README-short.txt
Normal file
1
docker/server/README.src/README-short.txt
Normal file
@ -0,0 +1 @@
|
||||
ClickHouse is the fastest and most resource efficient OSS database for real-time apps and analytics.
|
||||
170
docker/server/README.src/content.md
Normal file
170
docker/server/README.src/content.md
Normal file
@ -0,0 +1,170 @@
|
||||
# ClickHouse Server Docker Image
|
||||
|
||||
## What is ClickHouse?
|
||||
|
||||
%%LOGO%%
|
||||
|
||||
ClickHouse is an open-source column-oriented DBMS (columnar database management system) for online analytical processing (OLAP) that allows users to generate analytical reports using SQL queries in real-time.
|
||||
|
||||
ClickHouse works 100-1000x faster than traditional database management systems, and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second. With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance.
|
||||
|
||||
For more information and documentation see https://clickhouse.com/.
|
||||
|
||||
<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
|
||||
## Versions
|
||||
|
||||
- The `latest` tag points to the latest release of the latest stable branch.
|
||||
- Branch tags like `22.2` point to the latest release of the corresponding branch.
|
||||
- Full version tags like `22.2.3.5` point to the corresponding release.
|
||||
- The tag `head` is built from the latest commit to the default branch.
|
||||
- Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
|
||||
|
||||
<!-- REMOVE UNTIL HERE -->
|
||||
### Compatibility
|
||||
|
||||
- The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
|
||||
- The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A).
|
||||
- Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run --security-opt seccomp=unconfined` instead, however that has security implications.
|
||||
|
||||
## How to use this image
|
||||
|
||||
### start server instance
|
||||
|
||||
```bash
|
||||
docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
|
||||
```
|
||||
|
||||
By default, ClickHouse will be accessible only via the Docker network. See the **networking** section below.
|
||||
|
||||
By default, starting above server instance will be run as the `default` user without password.
|
||||
|
||||
### connect to it from a native client
|
||||
|
||||
```bash
|
||||
docker run -it --rm --link some-clickhouse-server:clickhouse-server --entrypoint clickhouse-client %%IMAGE%% --host clickhouse-server
|
||||
# OR
|
||||
docker exec -it some-clickhouse-server clickhouse-client
|
||||
```
|
||||
|
||||
More information about the [ClickHouse client](https://clickhouse.com/docs/en/interfaces/cli/).
|
||||
|
||||
### connect to it using curl
|
||||
|
||||
```bash
|
||||
echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server buildpack-deps:curl curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
|
||||
```
|
||||
|
||||
More information about the [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/).
|
||||
|
||||
### stopping / removing the container
|
||||
|
||||
```bash
|
||||
docker stop some-clickhouse-server
|
||||
docker rm some-clickhouse-server
|
||||
```
|
||||
|
||||
### networking
|
||||
|
||||
You can expose your ClickHouse running in docker by [mapping a particular port](https://docs.docker.com/config/containers/container-networking/) from inside the container using host ports:
|
||||
|
||||
```bash
|
||||
docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
|
||||
echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @-
|
||||
```
|
||||
|
||||
`22.6.3.35`
|
||||
|
||||
Or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
|
||||
|
||||
```bash
|
||||
docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
|
||||
echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @-
|
||||
```
|
||||
|
||||
`22.6.3.35`
|
||||
|
||||
### Volumes
|
||||
|
||||
Typically you may want to mount the following folders inside your container to achieve persistency:
|
||||
|
||||
- `/var/lib/clickhouse/` - main folder where ClickHouse stores the data
|
||||
- `/var/log/clickhouse-server/` - logs
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
-v "$PWD/ch_data:/var/lib/clickhouse/" \
|
||||
-v "$PWD/ch_logs:/var/log/clickhouse-server/" \
|
||||
--name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
|
||||
```
|
||||
|
||||
You may also want to mount:
|
||||
|
||||
- `/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustments
|
||||
- `/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustments
|
||||
- `/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below).
|
||||
|
||||
### Linux capabilities
|
||||
|
||||
ClickHouse has some advanced functionality, which requires enabling several [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html).
|
||||
|
||||
They are optional and can be enabled using the following [docker command-line arguments](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities):
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--cap-add=SYS_NICE --cap-add=NET_ADMIN --cap-add=IPC_LOCK \
|
||||
--name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
|
||||
```
|
||||
|
||||
Read more in [knowledge base](https://clickhouse.com/docs/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker).
|
||||
|
||||
## Configuration
|
||||
|
||||
The container exposes port 8123 for the [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and port 9000 for the [native client](https://clickhouse.com/docs/en/interfaces/tcp/).
|
||||
|
||||
ClickHouse configuration is represented with a file "config.xml" ([documentation](https://clickhouse.com/docs/en/operations/configuration_files/))
|
||||
|
||||
### Start server instance with custom configuration
|
||||
|
||||
```bash
|
||||
docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml %%IMAGE%%
|
||||
```
|
||||
|
||||
### Start server as custom user
|
||||
|
||||
```bash
|
||||
# $PWD/data/clickhouse should exist and be owned by current user
|
||||
docker run --rm --user "${UID}:${GID}" --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" %%IMAGE%%
|
||||
```
|
||||
|
||||
When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start.
|
||||
|
||||
### Start server from root (useful in case of enabled user namespace)
|
||||
|
||||
```bash
|
||||
docker run --rm -e CLICKHOUSE_RUN_AS_ROOT=1 --name clickhouse-server-userns -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" %%IMAGE%%
|
||||
```
|
||||
|
||||
### How to create default database and user on starting
|
||||
|
||||
Sometimes you may want to create a user (user named `default` is used by default) and database on a container start. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT` and `CLICKHOUSE_PASSWORD`:
|
||||
|
||||
```bash
|
||||
docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=password -p 9000:9000/tcp %%IMAGE%%
|
||||
```
|
||||
|
||||
## How to extend this image
|
||||
|
||||
To perform additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb`, it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service.
|
||||
Also, you can provide environment variables `CLICKHOUSE_USER` & `CLICKHOUSE_PASSWORD` that will be used for clickhouse-client during initialization.
|
||||
|
||||
For example, to add an additional user and database, add the following to `/docker-entrypoint-initdb.d/init-db.sh`:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
clickhouse client -n <<-EOSQL
|
||||
CREATE DATABASE docker;
|
||||
CREATE TABLE docker.docker (x Int32) ENGINE = Log;
|
||||
EOSQL
|
||||
```
|
||||
1
docker/server/README.src/github-repo
Normal file
1
docker/server/README.src/github-repo
Normal file
@ -0,0 +1 @@
|
||||
https://github.com/ClickHouse/ClickHouse
|
||||
1
docker/server/README.src/license.md
Normal file
1
docker/server/README.src/license.md
Normal file
@ -0,0 +1 @@
|
||||
View [license information](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) for the software contained in this image.
|
||||
43
docker/server/README.src/logo.svg
Normal file
43
docker/server/README.src/logo.svg
Normal file
@ -0,0 +1,43 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 616 616">
|
||||
<defs>
|
||||
<style>
|
||||
.cls-1 {
|
||||
clip-path: url(#clippath);
|
||||
}
|
||||
|
||||
.cls-2 {
|
||||
fill: none;
|
||||
}
|
||||
|
||||
.cls-2, .cls-3, .cls-4 {
|
||||
stroke-width: 0px;
|
||||
}
|
||||
|
||||
.cls-3 {
|
||||
fill: #1e1e1e;
|
||||
}
|
||||
|
||||
.cls-4 {
|
||||
fill: #faff69;
|
||||
}
|
||||
</style>
|
||||
<clipPath id="clippath">
|
||||
<rect class="cls-2" x="83.23" y="71.73" width="472.55" height="472.55"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
<g id="Layer_2" data-name="Layer 2">
|
||||
<rect class="cls-4" width="616" height="616"/>
|
||||
</g>
|
||||
<g id="Layer_1" data-name="Layer 1">
|
||||
<g class="cls-1">
|
||||
<g>
|
||||
<path class="cls-3" d="m120.14,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
|
||||
<path class="cls-3" d="m208.75,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
|
||||
<path class="cls-3" d="m297.35,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
|
||||
<path class="cls-3" d="m385.94,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
|
||||
<path class="cls-3" d="m474.56,268.36c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.65,2.09,4.65,4.66v79.28c0,2.57-2.09,4.66-4.65,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66v-79.28Z"/>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.7 KiB |
1
docker/server/README.src/maintainer.md
Normal file
1
docker/server/README.src/maintainer.md
Normal file
@ -0,0 +1 @@
|
||||
[ClickHouse Inc.](%%GITHUB-REPO%%)
|
||||
7
docker/server/README.src/metadata.json
Normal file
7
docker/server/README.src/metadata.json
Normal file
@ -0,0 +1,7 @@
|
||||
{
|
||||
"hub": {
|
||||
"categories": [
|
||||
"databases-and-storage"
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -4,17 +4,28 @@ set -eo pipefail
|
||||
shopt -s nullglob
|
||||
|
||||
DO_CHOWN=1
|
||||
if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then
|
||||
if [[ "${CLICKHOUSE_RUN_AS_ROOT:=0}" = "1" || "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]]; then
|
||||
DO_CHOWN=0
|
||||
fi
|
||||
|
||||
CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
|
||||
CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
|
||||
# CLICKHOUSE_UID and CLICKHOUSE_GID are kept for backward compatibility, but deprecated
|
||||
# One must use either "docker run --user" or CLICKHOUSE_RUN_AS_ROOT=1 to run the process as
|
||||
# FIXME: Remove ALL CLICKHOUSE_UID CLICKHOUSE_GID before 25.3
|
||||
if [[ "${CLICKHOUSE_UID:-}" || "${CLICKHOUSE_GID:-}" ]]; then
|
||||
echo 'WARNING: Support for CLICKHOUSE_UID/CLICKHOUSE_GID will be removed in a couple of releases.' >&2
|
||||
echo 'WARNING: Either use a proper "docker run --user=xxx:xxxx" argument instead of CLICKHOUSE_UID/CLICKHOUSE_GID' >&2
|
||||
echo 'WARNING: or set "CLICKHOUSE_RUN_AS_ROOT=1" ENV to run the clickhouse-server as root:root' >&2
|
||||
fi
|
||||
|
||||
# support --user
|
||||
if [ "$(id -u)" = "0" ]; then
|
||||
USER=$CLICKHOUSE_UID
|
||||
GROUP=$CLICKHOUSE_GID
|
||||
# support `docker run --user=xxx:xxxx`
|
||||
if [[ "$(id -u)" = "0" ]]; then
|
||||
if [[ "$CLICKHOUSE_RUN_AS_ROOT" = 1 ]]; then
|
||||
USER=0
|
||||
GROUP=0
|
||||
else
|
||||
USER="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
|
||||
GROUP="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
|
||||
fi
|
||||
else
|
||||
USER="$(id -u)"
|
||||
GROUP="$(id -g)"
|
||||
@ -55,14 +66,14 @@ function create_directory_and_do_chown() {
|
||||
[ -z "$dir" ] && return
|
||||
# ensure directories exist
|
||||
if [ "$DO_CHOWN" = "1" ]; then
|
||||
mkdir="mkdir"
|
||||
mkdir=( mkdir )
|
||||
else
|
||||
# if DO_CHOWN=0 it means that the system does not map root user to "admin" permissions
|
||||
# it mainly happens on NFS mounts where root==nobody for security reasons
|
||||
# thus mkdir MUST run with user id/gid and not from nobody that has zero permissions
|
||||
mkdir="/usr/bin/clickhouse su "${USER}:${GROUP}" mkdir"
|
||||
mkdir=( clickhouse su "${USER}:${GROUP}" mkdir )
|
||||
fi
|
||||
if ! $mkdir -p "$dir"; then
|
||||
if ! "${mkdir[@]}" -p "$dir"; then
|
||||
echo "Couldn't create necessary directory: $dir"
|
||||
exit 1
|
||||
fi
|
||||
@ -143,7 +154,7 @@ if [ -n "${RUN_INITDB_SCRIPTS}" ]; then
|
||||
fi
|
||||
|
||||
# Listen only on localhost until the initialization is done
|
||||
/usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 &
|
||||
clickhouse su "${USER}:${GROUP}" clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 &
|
||||
pid="$!"
|
||||
|
||||
# check if clickhouse is ready to accept connections
|
||||
@ -151,7 +162,7 @@ if [ -n "${RUN_INITDB_SCRIPTS}" ]; then
|
||||
tries=${CLICKHOUSE_INIT_TIMEOUT:-1000}
|
||||
while ! wget --spider --no-check-certificate -T 1 -q "$URL" 2>/dev/null; do
|
||||
if [ "$tries" -le "0" ]; then
|
||||
echo >&2 'ClickHouse init process failed.'
|
||||
echo >&2 'ClickHouse init process timeout.'
|
||||
exit 1
|
||||
fi
|
||||
tries=$(( tries-1 ))
|
||||
@ -203,18 +214,8 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
|
||||
CLICKHOUSE_WATCHDOG_ENABLE=${CLICKHOUSE_WATCHDOG_ENABLE:-0}
|
||||
export CLICKHOUSE_WATCHDOG_ENABLE
|
||||
|
||||
# An option for easy restarting and replacing clickhouse-server in a container, especially in Kubernetes.
|
||||
# For example, you can replace the clickhouse-server binary to another and restart it while keeping the container running.
|
||||
if [[ "${CLICKHOUSE_DOCKER_RESTART_ON_EXIT:-0}" -eq "1" ]]; then
|
||||
while true; do
|
||||
# This runs the server as a child process of the shell script:
|
||||
/usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@" ||:
|
||||
echo >&2 'ClickHouse Server exited, and the environment variable CLICKHOUSE_DOCKER_RESTART_ON_EXIT is set to 1. Restarting the server.'
|
||||
done
|
||||
else
|
||||
# This replaces the shell script with the server:
|
||||
exec /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@"
|
||||
fi
|
||||
# This replaces the shell script with the server:
|
||||
exec clickhouse su "${USER}:${GROUP}" clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@"
|
||||
fi
|
||||
|
||||
# Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image
|
||||
|
||||
@ -25,7 +25,7 @@ EXTRA_COLUMNS_EXPRESSION_TRACE_LOG="${EXTRA_COLUMNS_EXPRESSION}, arrayMap(x -> d
|
||||
|
||||
# coverage_log needs more columns for symbolization, but only symbol names (the line numbers are too heavy to calculate)
|
||||
EXTRA_COLUMNS_COVERAGE_LOG="${EXTRA_COLUMNS} symbols Array(LowCardinality(String)), "
|
||||
EXTRA_COLUMNS_EXPRESSION_COVERAGE_LOG="${EXTRA_COLUMNS_EXPRESSION}, arrayMap(x -> demangle(addressToSymbol(x)), coverage)::Array(LowCardinality(String)) AS symbols"
|
||||
EXTRA_COLUMNS_EXPRESSION_COVERAGE_LOG="${EXTRA_COLUMNS_EXPRESSION}, arrayDistinct(arrayMap(x -> demangle(addressToSymbol(x)), coverage))::Array(LowCardinality(String)) AS symbols"
|
||||
|
||||
|
||||
function __set_connection_args
|
||||
|
||||
@ -33,8 +33,6 @@ RUN apt-get update \
|
||||
COPY requirements.txt /
|
||||
RUN pip3 install --no-cache-dir -r /requirements.txt
|
||||
|
||||
ENV FUZZER_ARGS="-max_total_time=60"
|
||||
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
# docker run --network=host --volume <workspace>:/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
# Since right now we can't set volumes to the docker during build, we split building container in stages:
|
||||
# 1. build base container
|
||||
# 2. run base conatiner with mounted volumes
|
||||
# 3. commit container as image
|
||||
FROM ubuntu:20.04 as clickhouse-test-runner-base
|
||||
|
||||
# A volume where directory with clickhouse packages to be mounted,
|
||||
# for later installing.
|
||||
VOLUME /packages
|
||||
|
||||
CMD apt-get update ;\
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
apt install -y /packages/clickhouse-common-static_*.deb \
|
||||
/packages/clickhouse-client_*.deb \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
@ -28,7 +28,7 @@ COPY requirements.txt /
|
||||
RUN pip3 install --no-cache-dir -r requirements.txt
|
||||
|
||||
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
ENV LC_ALL=en_US.UTF-8
|
||||
|
||||
# Architecture of the image when BuildKit/buildx is used
|
||||
ARG TARGETARCH
|
||||
|
||||
@ -12,6 +12,7 @@ charset-normalizer==3.3.2
|
||||
click==8.1.7
|
||||
codespell==2.2.1
|
||||
cryptography==43.0.1
|
||||
datacompy==0.7.3
|
||||
Deprecated==1.2.14
|
||||
dill==0.3.8
|
||||
flake8==4.0.1
|
||||
@ -23,6 +24,7 @@ mccabe==0.6.1
|
||||
multidict==6.0.5
|
||||
mypy==1.8.0
|
||||
mypy-extensions==1.0.0
|
||||
pandas==2.2.3
|
||||
packaging==24.1
|
||||
pathspec==0.9.0
|
||||
pip==24.1.1
|
||||
|
||||
412
docs/changelogs/v24.10.1.2812-stable.md
Normal file
412
docs/changelogs/v24.10.1.2812-stable.md
Normal file
File diff suppressed because one or more lines are too long
61
docs/changelogs/v24.10.2.80-stable.md
Normal file
61
docs/changelogs/v24.10.2.80-stable.md
Normal file
@ -0,0 +1,61 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.10.2.80-stable (96b80057159) FIXME as compared to v24.10.1.2812-stable (9cd0a3738d5)
|
||||
|
||||
#### Backward Incompatible Change
|
||||
* Backported in [#71363](https://github.com/ClickHouse/ClickHouse/issues/71363): Fix possible error `No such file or directory` due to unescaped special symbols in files for JSON subcolumns. [#71182](https://github.com/ClickHouse/ClickHouse/pull/71182) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
|
||||
#### Performance Improvement
|
||||
* Backported in [#71852](https://github.com/ClickHouse/ClickHouse/issues/71852): Improve the performance and accuracy of system.query_metric_log collection interval by reducing the critical region. [#71473](https://github.com/ClickHouse/ClickHouse/pull/71473) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#71495](https://github.com/ClickHouse/ClickHouse/issues/71495): Enable `parallel_replicas_local_plan` by default. Building a full-fledged local plan on the query initiator improves parallel replicas performance with less resource consumption, provides opportunities to apply more query optimizations. [#70171](https://github.com/ClickHouse/ClickHouse/pull/70171) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Backported in [#71985](https://github.com/ClickHouse/ClickHouse/issues/71985): Fixes RIGHT / FULL joins in queries with parallel replicas. Now, RIGHT joins can be executed with parallel replicas (right table reading is distributed). FULL joins can't be parallelized among nodes, - executed locally. [#71162](https://github.com/ClickHouse/ClickHouse/pull/71162) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Backported in [#71670](https://github.com/ClickHouse/ClickHouse/issues/71670): When user/group is given as ID, the `clickhouse su` fails. This patch fixes it to accept `UID:GID` as well. ### Documentation entry for user-facing changes. [#71626](https://github.com/ClickHouse/ClickHouse/pull/71626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71940](https://github.com/ClickHouse/ClickHouse/issues/71940): Update `HostResolver` 3 times in a `history` period. [#71863](https://github.com/ClickHouse/ClickHouse/pull/71863) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Backported in [#71922](https://github.com/ClickHouse/ClickHouse/issues/71922): Allow_reorder_prewhere_conditions is on by default with old compatibility settings. [#71867](https://github.com/ClickHouse/ClickHouse/pull/71867) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#71588](https://github.com/ClickHouse/ClickHouse/issues/71588): Fix mismatched aggreage function name of quantileExactWeightedInterpolated. The bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/69619. cc @Algunenano. [#71168](https://github.com/ClickHouse/ClickHouse/pull/71168) ([李扬](https://github.com/taiyang-li)).
|
||||
* Backported in [#71357](https://github.com/ClickHouse/ClickHouse/issues/71357): Fix bad_weak_ptr exception with Dynamic in functions comparison. [#71183](https://github.com/ClickHouse/ClickHouse/pull/71183) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71467](https://github.com/ClickHouse/ClickHouse/issues/71467): Fix bug of memory usage increase if enable_filesystem_cache=1, but disk in storage configuration did not have any cache configuration. [#71261](https://github.com/ClickHouse/ClickHouse/pull/71261) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#71355](https://github.com/ClickHouse/ClickHouse/issues/71355): Fix possible error "Cannot read all data" erros during deserialization of LowCardinality dictionary from Dynamic column. [#71299](https://github.com/ClickHouse/ClickHouse/pull/71299) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71324](https://github.com/ClickHouse/ClickHouse/issues/71324): Fix incomplete cleanup of parallel output format in the client. [#71304](https://github.com/ClickHouse/ClickHouse/pull/71304) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71466](https://github.com/ClickHouse/ClickHouse/issues/71466): Added missing unescaping in named collections. Without fix clickhouse-server can't start. [#71308](https://github.com/ClickHouse/ClickHouse/pull/71308) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Backported in [#71393](https://github.com/ClickHouse/ClickHouse/issues/71393): Fix inconsistent AST formatting when granting wrong wildcard grants [#71309](https://github.com/ClickHouse/ClickHouse/issues/71309). [#71332](https://github.com/ClickHouse/ClickHouse/pull/71332) ([pufit](https://github.com/pufit)).
|
||||
* Backported in [#71379](https://github.com/ClickHouse/ClickHouse/issues/71379): Add try/catch to data parts destructors to avoid terminate. [#71364](https://github.com/ClickHouse/ClickHouse/pull/71364) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#71751](https://github.com/ClickHouse/ClickHouse/issues/71751): Check suspicious and experimental types in JSON type hints. [#71369](https://github.com/ClickHouse/ClickHouse/pull/71369) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71451](https://github.com/ClickHouse/ClickHouse/issues/71451): Start memory worker thread on non-Linux OS too (fixes [#71051](https://github.com/ClickHouse/ClickHouse/issues/71051)). [#71384](https://github.com/ClickHouse/ClickHouse/pull/71384) ([Alexandre Snarskii](https://github.com/snar)).
|
||||
* Backported in [#71608](https://github.com/ClickHouse/ClickHouse/issues/71608): Fix error Invalid number of rows in Chunk with Variant column. [#71388](https://github.com/ClickHouse/ClickHouse/pull/71388) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71493](https://github.com/ClickHouse/ClickHouse/issues/71493): Fix crash in `mongodb` table function when passing wrong arguments (e.g. `NULL`). [#71426](https://github.com/ClickHouse/ClickHouse/pull/71426) ([Vladimir Cherkasov](https://github.com/vdimir)).
|
||||
* Backported in [#71815](https://github.com/ClickHouse/ClickHouse/issues/71815): Fix crash with optimize_rewrite_array_exists_to_has. [#71432](https://github.com/ClickHouse/ClickHouse/pull/71432) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71521](https://github.com/ClickHouse/ClickHouse/issues/71521): Fix possible error `Argument for function must be constant` (old analyzer) in case when arrayJoin can apparently appear in `WHERE` condition. Regression after https://github.com/ClickHouse/ClickHouse/pull/65414. [#71476](https://github.com/ClickHouse/ClickHouse/pull/71476) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#71555](https://github.com/ClickHouse/ClickHouse/issues/71555): Prevent crash in SortCursor with 0 columns (old analyzer). [#71494](https://github.com/ClickHouse/ClickHouse/pull/71494) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71618](https://github.com/ClickHouse/ClickHouse/issues/71618): Analyzer fix when query inside materialized view uses IN with CTE. Closes [#65598](https://github.com/ClickHouse/ClickHouse/issues/65598). [#71538](https://github.com/ClickHouse/ClickHouse/pull/71538) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Backported in [#71570](https://github.com/ClickHouse/ClickHouse/issues/71570): Avoid crash when using a UDF in a constraint. [#71541](https://github.com/ClickHouse/ClickHouse/pull/71541) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71646](https://github.com/ClickHouse/ClickHouse/issues/71646): Return 0 or default char instead of throwing an error in bitShift functions in case of out of bounds. [#71580](https://github.com/ClickHouse/ClickHouse/pull/71580) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#71880](https://github.com/ClickHouse/ClickHouse/issues/71880): Fix LOGICAL_ERROR when doing ALTER with empty tuple. This fixes [#71647](https://github.com/ClickHouse/ClickHouse/issues/71647). [#71679](https://github.com/ClickHouse/ClickHouse/pull/71679) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#71741](https://github.com/ClickHouse/ClickHouse/issues/71741): Don't transform constant set in predicates over partition columns in case of NOT IN operator. [#71695](https://github.com/ClickHouse/ClickHouse/pull/71695) ([Eduard Karacharov](https://github.com/korowa)).
|
||||
* Backported in [#72012](https://github.com/ClickHouse/ClickHouse/issues/72012): Fix exception for toDayOfWeek on WHERE condition with primary key of DateTime64 type. [#71849](https://github.com/ClickHouse/ClickHouse/pull/71849) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Backported in [#71897](https://github.com/ClickHouse/ClickHouse/issues/71897): Fixed filling of defaults after parsing into sparse columns. [#71854](https://github.com/ClickHouse/ClickHouse/pull/71854) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#71955](https://github.com/ClickHouse/ClickHouse/issues/71955): Fix data race between the progress indicator and the progress table in clickhouse-client. This issue is visible when FROM INFILE is used. Intercept keystrokes during INSERT queries to toggle progress table display. [#71901](https://github.com/ClickHouse/ClickHouse/pull/71901) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Backported in [#72006](https://github.com/ClickHouse/ClickHouse/issues/72006): Fix a crash in clickhouse-client syntax highlighting. Closes [#71864](https://github.com/ClickHouse/ClickHouse/issues/71864). [#71949](https://github.com/ClickHouse/ClickHouse/pull/71949) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#71692](https://github.com/ClickHouse/ClickHouse/issues/71692): Improve clickhouse-server Dockerfile.ubuntu. Deprecate `CLICKHOUSE_UID/CLICKHOUSE_GID` envs. Remove `CLICKHOUSE_DOCKER_RESTART_ON_EXIT` processing to complien requirements. Consistent `clickhouse/clickhouse-server/clickhouse-keeper` execution to not have it plain in one place and `/usr/bin/clickhouse*` in another. [#71573](https://github.com/ClickHouse/ClickHouse/pull/71573) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#71387](https://github.com/ClickHouse/ClickHouse/issues/71387): Remove bad test `test_system_replicated_fetches`. [#71071](https://github.com/ClickHouse/ClickHouse/pull/71071) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#71586](https://github.com/ClickHouse/ClickHouse/issues/71586): Fix `WITH TOTALS` in subquery with parallel replicas. [#71224](https://github.com/ClickHouse/ClickHouse/pull/71224) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Backported in [#71437](https://github.com/ClickHouse/ClickHouse/issues/71437): Ignore `No such key` exceptions in some cases. [#71236](https://github.com/ClickHouse/ClickHouse/pull/71236) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#71629](https://github.com/ClickHouse/ClickHouse/issues/71629): Fix compatibility with refreshable materialized views created by old clickhouse servers. [#71556](https://github.com/ClickHouse/ClickHouse/pull/71556) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Backported in [#71805](https://github.com/ClickHouse/ClickHouse/issues/71805): Fix issues we face on orphane backport branches and closed release PRs, when fake-master events are sent to the check DB. [#71782](https://github.com/ClickHouse/ClickHouse/pull/71782) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71832](https://github.com/ClickHouse/ClickHouse/issues/71832): Closes [#71780](https://github.com/ClickHouse/ClickHouse/issues/71780). [#71818](https://github.com/ClickHouse/ClickHouse/pull/71818) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#71840](https://github.com/ClickHouse/ClickHouse/issues/71840): The change has already been applied to https://github.com/docker-library/official-images/pull/17876. Backport it to every branch to have a proper `Dockerfile.ubuntu` there. [#71825](https://github.com/ClickHouse/ClickHouse/pull/71825) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
31
docs/changelogs/v24.3.13.40-lts.md
Normal file
31
docs/changelogs/v24.3.13.40-lts.md
Normal file
@ -0,0 +1,31 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.3.13.40-lts (7acabd77389) FIXME as compared to v24.3.12.75-lts (7cb5dff8019)
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#63976](https://github.com/ClickHouse/ClickHouse/issues/63976): Fix intersect parts when restart after drop range. [#63202](https://github.com/ClickHouse/ClickHouse/pull/63202) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Backported in [#71482](https://github.com/ClickHouse/ClickHouse/issues/71482): Fix `Content-Encoding` not sent in some compressed responses. [#64802](https://github.com/ClickHouse/ClickHouse/issues/64802). [#68975](https://github.com/ClickHouse/ClickHouse/pull/68975) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Backported in [#70451](https://github.com/ClickHouse/ClickHouse/issues/70451): Fix vrash during insertion into FixedString column in PostgreSQL engine. [#69584](https://github.com/ClickHouse/ClickHouse/pull/69584) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70619](https://github.com/ClickHouse/ClickHouse/issues/70619): Fix server segfault on creating a materialized view with two selects and an `INTERSECT`, e.g. `CREATE MATERIALIZED VIEW v0 AS (SELECT 1) INTERSECT (SELECT 1);`. [#70264](https://github.com/ClickHouse/ClickHouse/pull/70264) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Backported in [#70877](https://github.com/ClickHouse/ClickHouse/issues/70877): Fix table creation with `CREATE ... AS table_function()` with database `Replicated` and unavailable table function source on secondary replica. [#70511](https://github.com/ClickHouse/ClickHouse/pull/70511) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#70571](https://github.com/ClickHouse/ClickHouse/issues/70571): Ignore all output on async insert with `wait_for_async_insert=1`. Closes [#62644](https://github.com/ClickHouse/ClickHouse/issues/62644). [#70530](https://github.com/ClickHouse/ClickHouse/pull/70530) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Backported in [#71146](https://github.com/ClickHouse/ClickHouse/issues/71146): Ignore frozen_metadata.txt while traversing shadow directory from system.remote_data_paths. [#70590](https://github.com/ClickHouse/ClickHouse/pull/70590) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Backported in [#70682](https://github.com/ClickHouse/ClickHouse/issues/70682): Fix creation of stateful window functions on misaligned memory. [#70631](https://github.com/ClickHouse/ClickHouse/pull/70631) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71113](https://github.com/ClickHouse/ClickHouse/issues/71113): Fix a crash and a leak in AggregateFunctionGroupArraySorted. [#70820](https://github.com/ClickHouse/ClickHouse/pull/70820) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Backported in [#70990](https://github.com/ClickHouse/ClickHouse/issues/70990): Fix a logical error due to negative zeros in the two-level hash table. This closes [#70973](https://github.com/ClickHouse/ClickHouse/issues/70973). [#70979](https://github.com/ClickHouse/ClickHouse/pull/70979) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#71246](https://github.com/ClickHouse/ClickHouse/issues/71246): Fixed named sessions not being closed and hanging on forever under certain circumstances. [#70998](https://github.com/ClickHouse/ClickHouse/pull/70998) ([Márcio Martins](https://github.com/marcio-absmartly)).
|
||||
* Backported in [#71371](https://github.com/ClickHouse/ClickHouse/issues/71371): Add try/catch to data parts destructors to avoid terminate. [#71364](https://github.com/ClickHouse/ClickHouse/pull/71364) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#71594](https://github.com/ClickHouse/ClickHouse/issues/71594): Prevent crash in SortCursor with 0 columns (old analyzer). [#71494](https://github.com/ClickHouse/ClickHouse/pull/71494) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#71022](https://github.com/ClickHouse/ClickHouse/issues/71022): Fix dropping of file cache in CHECK query in case of enabled transactions. [#69256](https://github.com/ClickHouse/ClickHouse/pull/69256) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#70384](https://github.com/ClickHouse/ClickHouse/issues/70384): CI: Enable Integration Tests for backport PRs. [#70329](https://github.com/ClickHouse/ClickHouse/pull/70329) ([Max Kainov](https://github.com/maxknv)).
|
||||
* Backported in [#70538](https://github.com/ClickHouse/ClickHouse/issues/70538): Remove slow poll() logs in keeper. [#70508](https://github.com/ClickHouse/ClickHouse/pull/70508) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#70971](https://github.com/ClickHouse/ClickHouse/issues/70971): Limiting logging some lines about configs. [#70879](https://github.com/ClickHouse/ClickHouse/pull/70879) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
|
||||
31
docs/changelogs/v24.3.14.35-lts.md
Normal file
31
docs/changelogs/v24.3.14.35-lts.md
Normal file
@ -0,0 +1,31 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.3.14.35-lts (cfa4e62b775) FIXME as compared to v24.3.13.40-lts (7acabd77389)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#71711](https://github.com/ClickHouse/ClickHouse/issues/71711): CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)).
|
||||
* Backported in [#71662](https://github.com/ClickHouse/ClickHouse/issues/71662): When user/group is given as ID, the `clickhouse su` fails. This patch fixes it to accept `UID:GID` as well. ### Documentation entry for user-facing changes. [#71626](https://github.com/ClickHouse/ClickHouse/pull/71626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#65755](https://github.com/ClickHouse/ClickHouse/issues/65755): Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.`. [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#71600](https://github.com/ClickHouse/ClickHouse/issues/71600): Fix error Invalid number of rows in Chunk with Variant column. [#71388](https://github.com/ClickHouse/ClickHouse/pull/71388) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71842](https://github.com/ClickHouse/ClickHouse/issues/71842): Fix crash with optimize_rewrite_array_exists_to_has. [#71432](https://github.com/ClickHouse/ClickHouse/pull/71432) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71562](https://github.com/ClickHouse/ClickHouse/issues/71562): Avoid crash when using a UDF in a constraint. [#71541](https://github.com/ClickHouse/ClickHouse/pull/71541) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71731](https://github.com/ClickHouse/ClickHouse/issues/71731): Return 0 or default char instead of throwing an error in bitShift functions in case of out of bounds. [#71580](https://github.com/ClickHouse/ClickHouse/pull/71580) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#71697](https://github.com/ClickHouse/ClickHouse/issues/71697): Vendor in rust dependencies. [#62297](https://github.com/ClickHouse/ClickHouse/pull/62297) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71688](https://github.com/ClickHouse/ClickHouse/issues/71688): Improve clickhouse-server Dockerfile.ubuntu. Deprecate `CLICKHOUSE_UID/CLICKHOUSE_GID` envs. Remove `CLICKHOUSE_DOCKER_RESTART_ON_EXIT` processing to complien requirements. Consistent `clickhouse/clickhouse-server/clickhouse-keeper` execution to not have it plain in one place and `/usr/bin/clickhouse*` in another. [#71573](https://github.com/ClickHouse/ClickHouse/pull/71573) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#71808](https://github.com/ClickHouse/ClickHouse/issues/71808): Fix issues we face on orphane backport branches and closed release PRs, when fake-master events are sent to the check DB. [#71782](https://github.com/ClickHouse/ClickHouse/pull/71782) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71834](https://github.com/ClickHouse/ClickHouse/issues/71834): The change has already been applied to https://github.com/docker-library/official-images/pull/17876. Backport it to every branch to have a proper `Dockerfile.ubuntu` there. [#71825](https://github.com/ClickHouse/ClickHouse/pull/71825) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Fix bitShift test after backport. [#71861](https://github.com/ClickHouse/ClickHouse/pull/71861) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Revert "Merge pull request [#71861](https://github.com/ClickHouse/ClickHouse/issues/71861) from pamarcos/fix-bitshift-test". [#71871](https://github.com/ClickHouse/ClickHouse/pull/71871) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
|
||||
50
docs/changelogs/v24.8.6.70-lts.md
Normal file
50
docs/changelogs/v24.8.6.70-lts.md
Normal file
File diff suppressed because one or more lines are too long
37
docs/changelogs/v24.8.7.41-lts.md
Normal file
37
docs/changelogs/v24.8.7.41-lts.md
Normal file
@ -0,0 +1,37 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.8.7.41-lts (e28553d4f2b) FIXME as compared to v24.8.6.70-lts (ddb8c219771)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#71713](https://github.com/ClickHouse/ClickHouse/issues/71713): CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)).
|
||||
* Backported in [#71666](https://github.com/ClickHouse/ClickHouse/issues/71666): When user/group is given as ID, the `clickhouse su` fails. This patch fixes it to accept `UID:GID` as well. ### Documentation entry for user-facing changes. [#71626](https://github.com/ClickHouse/ClickHouse/pull/71626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71936](https://github.com/ClickHouse/ClickHouse/issues/71936): Update `HostResolver` 3 times in a `history` period. [#71863](https://github.com/ClickHouse/ClickHouse/pull/71863) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#71486](https://github.com/ClickHouse/ClickHouse/issues/71486): Fix `Content-Encoding` not sent in some compressed responses. [#64802](https://github.com/ClickHouse/ClickHouse/issues/64802). [#68975](https://github.com/ClickHouse/ClickHouse/pull/68975) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Backported in [#71462](https://github.com/ClickHouse/ClickHouse/issues/71462): Added missing unescaping in named collections. Without fix clickhouse-server can't start. [#71308](https://github.com/ClickHouse/ClickHouse/pull/71308) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Backported in [#71747](https://github.com/ClickHouse/ClickHouse/issues/71747): Check suspicious and experimental types in JSON type hints. [#71369](https://github.com/ClickHouse/ClickHouse/pull/71369) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71604](https://github.com/ClickHouse/ClickHouse/issues/71604): Fix error Invalid number of rows in Chunk with Variant column. [#71388](https://github.com/ClickHouse/ClickHouse/pull/71388) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71826](https://github.com/ClickHouse/ClickHouse/issues/71826): Fix crash with optimize_rewrite_array_exists_to_has. [#71432](https://github.com/ClickHouse/ClickHouse/pull/71432) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71517](https://github.com/ClickHouse/ClickHouse/issues/71517): Fix possible error `Argument for function must be constant` (old analyzer) in case when arrayJoin can apparently appear in `WHERE` condition. Regression after https://github.com/ClickHouse/ClickHouse/pull/65414. [#71476](https://github.com/ClickHouse/ClickHouse/pull/71476) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#71551](https://github.com/ClickHouse/ClickHouse/issues/71551): Prevent crash in SortCursor with 0 columns (old analyzer). [#71494](https://github.com/ClickHouse/ClickHouse/pull/71494) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71614](https://github.com/ClickHouse/ClickHouse/issues/71614): Analyzer fix when query inside materialized view uses IN with CTE. Closes [#65598](https://github.com/ClickHouse/ClickHouse/issues/65598). [#71538](https://github.com/ClickHouse/ClickHouse/pull/71538) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Backported in [#71566](https://github.com/ClickHouse/ClickHouse/issues/71566): Avoid crash when using a UDF in a constraint. [#71541](https://github.com/ClickHouse/ClickHouse/pull/71541) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71727](https://github.com/ClickHouse/ClickHouse/issues/71727): Return 0 or default char instead of throwing an error in bitShift functions in case of out of bounds. [#71580](https://github.com/ClickHouse/ClickHouse/pull/71580) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#71876](https://github.com/ClickHouse/ClickHouse/issues/71876): Fix LOGICAL_ERROR when doing ALTER with empty tuple. This fixes [#71647](https://github.com/ClickHouse/ClickHouse/issues/71647). [#71679](https://github.com/ClickHouse/ClickHouse/pull/71679) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#71737](https://github.com/ClickHouse/ClickHouse/issues/71737): Don't transform constant set in predicates over partition columns in case of NOT IN operator. [#71695](https://github.com/ClickHouse/ClickHouse/pull/71695) ([Eduard Karacharov](https://github.com/korowa)).
|
||||
* Backported in [#72002](https://github.com/ClickHouse/ClickHouse/issues/72002): Fix a crash in clickhouse-client syntax highlighting. Closes [#71864](https://github.com/ClickHouse/ClickHouse/issues/71864). [#71949](https://github.com/ClickHouse/ClickHouse/pull/71949) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#71690](https://github.com/ClickHouse/ClickHouse/issues/71690): Improve clickhouse-server Dockerfile.ubuntu. Deprecate `CLICKHOUSE_UID/CLICKHOUSE_GID` envs. Remove `CLICKHOUSE_DOCKER_RESTART_ON_EXIT` processing to complien requirements. Consistent `clickhouse/clickhouse-server/clickhouse-keeper` execution to not have it plain in one place and `/usr/bin/clickhouse*` in another. [#71573](https://github.com/ClickHouse/ClickHouse/pull/71573) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#71801](https://github.com/ClickHouse/ClickHouse/issues/71801): Fix issues we face on orphane backport branches and closed release PRs, when fake-master events are sent to the check DB. [#71782](https://github.com/ClickHouse/ClickHouse/pull/71782) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71836](https://github.com/ClickHouse/ClickHouse/issues/71836): The change has already been applied to https://github.com/docker-library/official-images/pull/17876. Backport it to every branch to have a proper `Dockerfile.ubuntu` there. [#71825](https://github.com/ClickHouse/ClickHouse/pull/71825) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
88
docs/changelogs/v24.9.3.128-stable.md
Normal file
88
docs/changelogs/v24.9.3.128-stable.md
Normal file
File diff suppressed because one or more lines are too long
@ -4,9 +4,13 @@ sidebar_position: 50
|
||||
sidebar_label: EmbeddedRocksDB
|
||||
---
|
||||
|
||||
import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
|
||||
|
||||
# EmbeddedRocksDB Engine
|
||||
|
||||
This engine allows integrating ClickHouse with [rocksdb](http://rocksdb.org/).
|
||||
<CloudNotSupportedBadge />
|
||||
|
||||
This engine allows integrating ClickHouse with [RocksDB](http://rocksdb.org/).
|
||||
|
||||
## Creating a Table {#creating-a-table}
|
||||
|
||||
|
||||
@ -290,6 +290,7 @@ The following settings can be specified in configuration file for given endpoint
|
||||
- `expiration_window_seconds` — Grace period for checking if expiration-based credentials have expired. Optional, default value is `120`.
|
||||
- `no_sign_request` - Ignore all the credentials so requests are not signed. Useful for accessing public buckets.
|
||||
- `header` — Adds specified HTTP header to a request to given endpoint. Optional, can be specified multiple times.
|
||||
- `access_header` - Adds specified HTTP header to a request to given endpoint, in cases where there are no other credentials from another source.
|
||||
- `server_side_encryption_customer_key_base64` — If specified, required headers for accessing S3 objects with SSE-C encryption will be set. Optional.
|
||||
- `server_side_encryption_kms_key_id` - If specified, required headers for accessing S3 objects with [SSE-KMS encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html) will be set. If an empty string is specified, the AWS managed S3 key will be used. Optional.
|
||||
- `server_side_encryption_kms_encryption_context` - If specified alongside `server_side_encryption_kms_key_id`, the given encryption context header for SSE-KMS will be set. Optional.
|
||||
@ -320,6 +321,32 @@ The following settings can be specified in configuration file for given endpoint
|
||||
</s3>
|
||||
```
|
||||
|
||||
## Working with archives
|
||||
|
||||
Suppose that we have several archive files with following URIs on S3:
|
||||
|
||||
- 'https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m-2018-01-10.csv.zip'
|
||||
- 'https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m-2018-01-11.csv.zip'
|
||||
- 'https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m-2018-01-12.csv.zip'
|
||||
|
||||
Extracting data from these archives is possible using ::. Globs can be used both in the url part as well as in the part after :: (responsible for the name of a file inside the archive).
|
||||
|
||||
``` sql
|
||||
SELECT *
|
||||
FROM s3(
|
||||
'https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m-2018-01-1{0..2}.csv.zip :: *.csv'
|
||||
);
|
||||
```
|
||||
|
||||
:::note
|
||||
ClickHouse supports three archive formats:
|
||||
ZIP
|
||||
TAR
|
||||
7Z
|
||||
While ZIP and TAR archives can be accessed from any supported storage location, 7Z archives can only be read from the local filesystem where ClickHouse is installed.
|
||||
:::
|
||||
|
||||
|
||||
## Accessing public buckets
|
||||
|
||||
ClickHouse tries to fetch credentials from many different types of sources.
|
||||
|
||||
@ -122,7 +122,7 @@ Default value: `0`.
|
||||
|
||||
### s3queue_polling_min_timeout_ms {#polling_min_timeout_ms}
|
||||
|
||||
Minimal timeout before next polling (in milliseconds).
|
||||
Specifies the minimum time, in milliseconds, that ClickHouse waits before making the next polling attempt.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -132,7 +132,7 @@ Default value: `1000`.
|
||||
|
||||
### s3queue_polling_max_timeout_ms {#polling_max_timeout_ms}
|
||||
|
||||
Maximum timeout before next polling (in milliseconds).
|
||||
Defines the maximum time, in milliseconds, that ClickHouse waits before initiating the next polling attempt.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -142,7 +142,7 @@ Default value: `10000`.
|
||||
|
||||
### s3queue_polling_backoff_ms {#polling_backoff_ms}
|
||||
|
||||
Polling backoff (in milliseconds).
|
||||
Determines the additional wait time added to the previous polling interval when no new files are found. The next poll occurs after the sum of the previous interval and this backoff value, or the maximum interval, whichever is lower.
|
||||
|
||||
Possible values:
|
||||
|
||||
|
||||
@ -10,6 +10,11 @@ The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-fa
|
||||
|
||||
You can use `AggregatingMergeTree` tables for incremental data aggregation, including for aggregated materialized views.
|
||||
|
||||
You can see an example of how to use the AggregatingMergeTree and Aggregate functions in the below video:
|
||||
<div class='vimeo-container'>
|
||||
<iframe width="1030" height="579" src="https://www.youtube.com/embed/pryhI4F_zqQ" title="Aggregation States in ClickHouse" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
|
||||
</div>
|
||||
|
||||
The engine processes all columns with the following types:
|
||||
|
||||
## [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md)
|
||||
|
||||
@ -54,7 +54,7 @@ Parameters:
|
||||
- `distance_function`: either `L2Distance` (the [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance) - the length of a
|
||||
line between two points in Euclidean space), or `cosineDistance` (the [cosine
|
||||
distance](https://en.wikipedia.org/wiki/Cosine_similarity#Cosine_distance)- the angle between two non-zero vectors).
|
||||
- `quantization`: either `f64`, `f32`, `f16`, `bf16`, or `i8` for storing the vector with reduced precision (optional, default: `bf16`)
|
||||
- `quantization`: either `f64`, `f32`, `f16`, `bf16`, or `i8` for storing vectors with reduced precision (optional, default: `bf16`)
|
||||
- `hnsw_max_connections_per_layer`: the number of neighbors per HNSW graph node, also known as `M` in the [HNSW
|
||||
paper](https://doi.org/10.1109/TPAMI.2018.2889473) (optional, default: 32)
|
||||
- `hnsw_candidate_list_size_for_construction`: the size of the dynamic candidate list when constructing the HNSW graph, also known as
|
||||
@ -92,8 +92,8 @@ Vector similarity indexes currently support two distance functions:
|
||||
- `cosineDistance`, also called cosine similarity, is the cosine of the angle between two (non-zero) vectors
|
||||
([Wikipedia](https://en.wikipedia.org/wiki/Cosine_similarity)).
|
||||
|
||||
Vector similarity indexes allows storing the vectors in reduced precision formats. Supported scalar kinds are `f64`, `f32`, `f16` or `i8`.
|
||||
If no scalar kind was specified during index creation, `f16` is used as default.
|
||||
Vector similarity indexes allows storing the vectors in reduced precision formats. Supported scalar kinds are `f64`, `f32`, `f16`, `bf16`,
|
||||
and `i8`. If no scalar kind was specified during index creation, `bf16` is used as default.
|
||||
|
||||
For normalized data, `L2Distance` is usually a better choice, otherwise `cosineDistance` is recommended to compensate for scale. If no
|
||||
distance function was specified during index creation, `L2Distance` is used as default.
|
||||
|
||||
@ -684,8 +684,7 @@ If you perform the `SELECT` query between merges, you may get expired data. To a
|
||||
|
||||
**See Also**
|
||||
|
||||
- [ttl_only_drop_parts](/docs/en/operations/settings/settings.md/#ttl_only_drop_parts) setting
|
||||
|
||||
- [ttl_only_drop_parts](/docs/en/operations/settings/merge-tree-settings#ttl_only_drop_parts) setting
|
||||
|
||||
## Disk types
|
||||
|
||||
|
||||
@ -33,6 +33,21 @@ Then, generate the data. Parameter `-s` specifies the scale factor. For example,
|
||||
./dbgen -s 100
|
||||
```
|
||||
|
||||
Detailed table sizes with scale factor 100:
|
||||
|
||||
| Table | size (in rows) | size (compressed in ClickHouse) |
|
||||
|----------|----------------|---------------------------------|
|
||||
| nation | 25 | 2 kB |
|
||||
| region | 5 | 1 kB |
|
||||
| part | 20.000.000 | 895 MB |
|
||||
| supplier | 1.000.000 | 75 MB |
|
||||
| partsupp | 80.000.000 | 4.37 GB |
|
||||
| customer | 15.000.000 | 1.19 GB |
|
||||
| orders | 150.000.000 | 6.15 GB |
|
||||
| lineitem | 600.00.00 | 26.69 GB |
|
||||
|
||||
(Compressed sizes in ClickHouse are taken from `system.tables.total_bytes` and based on below table definitions.)
|
||||
|
||||
Now create tables in ClickHouse.
|
||||
|
||||
We stick as closely as possible to the rules of the TPC-H specification:
|
||||
@ -151,10 +166,37 @@ clickhouse-client --format_csv_delimiter '|' --query "INSERT INTO orders FORMAT
|
||||
clickhouse-client --format_csv_delimiter '|' --query "INSERT INTO lineitem FORMAT CSV" < lineitem.tbl
|
||||
```
|
||||
|
||||
The queries are generated by `./qgen -s <scaling_factor>`. Example queries for `s = 100`:
|
||||
:::note
|
||||
Instead of using tpch-kit and generating the tables by yourself, you can alternatively import the data from a public S3 bucket. Make sure
|
||||
to create empty tables first using above `CREATE` statements.
|
||||
|
||||
```sql
|
||||
-- Scaling factor 1
|
||||
INSERT INTO nation SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/nation.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO region SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/region.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO part SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/part.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO supplier SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/supplier.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO partsupp SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/partsupp.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO customer SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/customer.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO orders SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/orders.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO lineitem SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/lineitem.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
|
||||
-- Scaling factor 100
|
||||
INSERT INTO nation SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/nation.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO region SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/region.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO part SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/part.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO supplier SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/supplier.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO partsupp SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/partsupp.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO customer SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/customer.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO orders SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/orders.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
INSERT INTO lineitem SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/lineitem.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
|
||||
````
|
||||
:::
|
||||
|
||||
## Queries
|
||||
|
||||
The queries are generated by `./qgen -s <scaling_factor>`. Example queries for `s = 100`:
|
||||
|
||||
**Correctness**
|
||||
|
||||
The result of the queries agrees with the official results unless mentioned otherwise. To verify, generate a TPC-H database with scale
|
||||
|
||||
@ -23,6 +23,7 @@ functions in ClickHouse. The sample datasets include:
|
||||
- The [NYPD Complaint Data](../getting-started/example-datasets/nypd_complaint_data.md) demonstrates how to use data inference to simplify creating tables
|
||||
- The ["What's on the Menu?" dataset](../getting-started/example-datasets/menus.md) has an example of denormalizing data
|
||||
- The [Laion dataset](../getting-started/example-datasets/laion.md) has an example of [Approximate nearest neighbor search indexes](../engines/table-engines/mergetree-family/annindexes.md) usage
|
||||
- The [TPC-H](../getting-started/example-datasets/tpch.md), [TPC-DS](../getting-started/example-datasets/tpcds.md), and [Star Schema (SSB)](../getting-started/example-datasets/star-schema.md) industry benchmarks for analytics databases
|
||||
- [Getting Data Into ClickHouse - Part 1](https://clickhouse.com/blog/getting-data-into-clickhouse-part-1) provides examples of defining a schema and loading a small Hacker News dataset
|
||||
- [Getting Data Into ClickHouse - Part 3 - Using S3](https://clickhouse.com/blog/getting-data-into-clickhouse-part-3-s3) has examples of loading data from s3
|
||||
- [Generating random data in ClickHouse](https://clickhouse.com/blog/generating-random-test-distribution-data-for-clickhouse) shows how to generate random data if none of the above fit your needs.
|
||||
|
||||
@ -16,7 +16,7 @@ You have four options for getting up and running with ClickHouse:
|
||||
- **[ClickHouse Cloud](https://clickhouse.com/cloud/):** The official ClickHouse as a service, - built by, maintained and supported by the creators of ClickHouse
|
||||
- **[Quick Install](#quick-install):** an easy-to-download binary for testing and developing with ClickHouse
|
||||
- **[Production Deployments](#available-installation-options):** ClickHouse can run on any Linux, FreeBSD, or macOS with x86-64, modern ARM (ARMv8.2-A up), or PowerPC64LE CPU architecture
|
||||
- **[Docker Image](https://hub.docker.com/r/clickhouse/clickhouse-server/):** use the official Docker image in Docker Hub
|
||||
- **[Docker Image](https://hub.docker.com/_/clickhouse):** use the official Docker image in Docker Hub
|
||||
|
||||
## ClickHouse Cloud
|
||||
|
||||
|
||||
@ -190,6 +190,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
|
||||
- `--config-file` – The name of the configuration file.
|
||||
- `--secure` – If specified, will connect to server over secure connection (TLS). You might need to configure your CA certificates in the [configuration file](#configuration_files). The available configuration settings are the same as for [server-side TLS configuration](../operations/server-configuration-parameters/settings.md#openssl).
|
||||
- `--history_file` — Path to a file containing command history.
|
||||
- `--history_max_entries` — Maximum number of entries in the history file. Default value: 1 000 000.
|
||||
- `--param_<name>` — Value for a [query with parameters](#cli-queries-with-parameters).
|
||||
- `--hardware-utilization` — Print hardware utilization information in progress bar.
|
||||
- `--print-profile-events` – Print `ProfileEvents` packets.
|
||||
|
||||
@ -9,7 +9,7 @@ sidebar_label: Prometheus protocols
|
||||
## Exposing metrics {#expose}
|
||||
|
||||
:::note
|
||||
ClickHouse Cloud does not currently support connecting to Prometheus. To be notified when this feature is supported, please contact support@clickhouse.com.
|
||||
If you are using ClickHouse Cloud, you can expose metrics to Prometheus using the [Prometheus Integration](/en/integrations/prometheus).
|
||||
:::
|
||||
|
||||
ClickHouse can expose its own metrics for scraping from Prometheus:
|
||||
|
||||
@ -65,6 +65,34 @@ sudo rm -f /etc/yum.repos.d/clickhouse.repo
|
||||
|
||||
After that follow the [install guide](../getting-started/install.md#from-rpm-packages)
|
||||
|
||||
### You Can't Run Docker Container
|
||||
|
||||
You are running a simple `docker run clickhouse/clickhouse-server` and it crashes with a stack trace similar to following:
|
||||
|
||||
```
|
||||
$ docker run -it clickhouse/clickhouse-server
|
||||
........
|
||||
2024.11.06 21:04:48.912036 [ 1 ] {} <Information> SentryWriter: Sending crash reports is disabled
|
||||
Poco::Exception. Code: 1000, e.code() = 0, System exception: cannot start thread, Stack trace (when copying this message, always include the lines below):
|
||||
|
||||
0. Poco::ThreadImpl::startImpl(Poco::SharedPtr<Poco::Runnable, Poco::ReferenceCounter, Poco::ReleasePolicy<Poco::Runnable>>) @ 0x00000000157c7b34
|
||||
1. Poco::Thread::start(Poco::Runnable&) @ 0x00000000157c8a0e
|
||||
2. BaseDaemon::initializeTerminationAndSignalProcessing() @ 0x000000000d267a14
|
||||
3. BaseDaemon::initialize(Poco::Util::Application&) @ 0x000000000d2652cb
|
||||
4. DB::Server::initialize(Poco::Util::Application&) @ 0x000000000d128b38
|
||||
5. Poco::Util::Application::run() @ 0x000000001581cfda
|
||||
6. DB::Server::run() @ 0x000000000d1288f0
|
||||
7. Poco::Util::ServerApplication::run(int, char**) @ 0x0000000015825e27
|
||||
8. mainEntryClickHouseServer(int, char**) @ 0x000000000d125b38
|
||||
9. main @ 0x0000000007ea4eee
|
||||
10. ? @ 0x00007f67ff946d90
|
||||
11. ? @ 0x00007f67ff946e40
|
||||
12. _start @ 0x00000000062e802e
|
||||
(version 24.10.1.2812 (official build))
|
||||
```
|
||||
|
||||
The reason is an old docker daemon with version lower than `20.10.10`. A way to fix it either upgrading it, or running `docker run [--privileged | --security-opt seccomp=unconfined]`. The latter has security implications.
|
||||
|
||||
## Connecting to the Server {#troubleshooting-accepts-no-connections}
|
||||
|
||||
Possible issues:
|
||||
|
||||
@ -25,9 +25,10 @@ Query caches can generally be viewed as transactionally consistent or inconsiste
|
||||
slowly enough that the database only needs to compute the report once (represented by the first `SELECT` query). Further queries can be
|
||||
served directly from the query cache. In this example, a reasonable validity period could be 30 min.
|
||||
|
||||
Transactionally inconsistent caching is traditionally provided by client tools or proxy packages interacting with the database. As a result,
|
||||
the same caching logic and configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side.
|
||||
This reduces maintenance effort and avoids redundancy.
|
||||
Transactionally inconsistent caching is traditionally provided by client tools or proxy packages (e.g.
|
||||
[chproxy](https://www.chproxy.org/configuration/caching/)) interacting with the database. As a result, the same caching logic and
|
||||
configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side. This reduces maintenance
|
||||
effort and avoids redundancy.
|
||||
|
||||
## Configuration Settings and Usage
|
||||
|
||||
@ -138,7 +139,10 @@ is only cached if the query runs longer than 5 seconds. It is also possible to s
|
||||
cached - for that use setting [query_cache_min_query_runs](settings/settings.md#query-cache-min-query-runs).
|
||||
|
||||
Entries in the query cache become stale after a certain time period (time-to-live). By default, this period is 60 seconds but a different
|
||||
value can be specified at session, profile or query level using setting [query_cache_ttl](settings/settings.md#query-cache-ttl).
|
||||
value can be specified at session, profile or query level using setting [query_cache_ttl](settings/settings.md#query-cache-ttl). The query
|
||||
cache evicts entries "lazily", i.e. when an entry becomes stale, it is not immediately removed from the cache. Instead, when a new entry
|
||||
is to be inserted into the query cache, the database checks whether the cache has enough free space for the new entry. If this is not the
|
||||
case, the database tries to remove all stale entries. If the cache still has not enough free space, the new entry is not inserted.
|
||||
|
||||
Entries in the query cache are compressed by default. This reduces the overall memory consumption at the cost of slower writes into / reads
|
||||
from the query cache. To disable compression, use setting [query_cache_compress_entries](settings/settings.md#query-cache-compress-entries).
|
||||
@ -188,14 +192,9 @@ Also, results of queries with non-deterministic functions are not cached by defa
|
||||
To force caching of results of queries with non-deterministic functions regardless, use setting
|
||||
[query_cache_nondeterministic_function_handling](settings/settings.md#query-cache-nondeterministic-function-handling).
|
||||
|
||||
Results of queries that involve system tables, e.g. `system.processes` or `information_schema.tables`, are not cached by default. To force
|
||||
caching of results of queries with system tables regardless, use setting
|
||||
[query_cache_system_table_handling](settings/settings.md#query-cache-system-table-handling).
|
||||
|
||||
:::note
|
||||
Prior to ClickHouse v23.11, setting 'query_cache_store_results_of_queries_with_nondeterministic_functions = 0 / 1' controlled whether
|
||||
results of queries with non-deterministic results were cached. In newer ClickHouse versions, this setting is obsolete and has no effect.
|
||||
:::
|
||||
Results of queries that involve system tables (e.g. [system.processes](system-tables/processes.md)` or
|
||||
[information_schema.tables](system-tables/information_schema.md)) are not cached by default. To force caching of results of queries with
|
||||
system tables regardless, use setting [query_cache_system_table_handling](settings/settings.md#query-cache-system-table-handling).
|
||||
|
||||
Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a
|
||||
row policy on a table by running the same query as another user B for whom no such policy exists. However, if necessary, cache entries can
|
||||
|
||||
@ -131,16 +131,6 @@ Type: UInt64
|
||||
|
||||
Default: 8
|
||||
|
||||
## background_pool_size
|
||||
|
||||
Sets the number of threads performing background merges and mutations for tables with MergeTree engines. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance.
|
||||
|
||||
Before changing it, please also take a look at related MergeTree settings, such as `number_of_free_entries_in_pool_to_lower_max_size_of_merge` and `number_of_free_entries_in_pool_to_execute_mutation`.
|
||||
|
||||
Type: UInt64
|
||||
|
||||
Default: 16
|
||||
|
||||
## background_schedule_pool_size
|
||||
|
||||
The maximum number of threads that will be used for constantly executing some lightweight periodic operations for replicated tables, Kafka streaming, and DNS cache updates.
|
||||
@ -607,6 +597,30 @@ If number of tables is greater than this value, server will throw an exception.
|
||||
<max_table_num_to_throw>400</max_table_num_to_throw>
|
||||
```
|
||||
|
||||
## max\_replicated\_table\_num\_to\_throw {#max-replicated-table-num-to-throw}
|
||||
If number of replicated tables is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.
|
||||
|
||||
**Example**
|
||||
```xml
|
||||
<max_replicated_table_num_to_throw>400</max_replicated_table_num_to_throw>
|
||||
```
|
||||
|
||||
## max\_dictionary\_num\_to\_throw {#max-dictionary-num-to-throw}
|
||||
If number of dictionaries is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.
|
||||
|
||||
**Example**
|
||||
```xml
|
||||
<max_dictionary_num_to_throw>400</max_dictionary_num_to_throw>
|
||||
```
|
||||
|
||||
## max\_view\_num\_to\_throw {#max-view-num-to-throw}
|
||||
If number of views is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.
|
||||
|
||||
**Example**
|
||||
```xml
|
||||
<max_view_num_to_throw>400</max_view_num_to_throw>
|
||||
```
|
||||
|
||||
## max\_database\_num\_to\_throw {#max-table-num-to-throw}
|
||||
If number of _database is greater than this value, server will throw an exception. 0 means no limitation.
|
||||
Default value: 0
|
||||
@ -1629,6 +1643,7 @@ You can specify the log format that will be outputted in the console log. Curren
|
||||
|
||||
```json
|
||||
{
|
||||
"date_time_utc": "2024-11-06T09:06:09Z",
|
||||
"date_time": "1650918987.180175",
|
||||
"thread_name": "#1",
|
||||
"thread_id": "254545",
|
||||
@ -3224,6 +3239,34 @@ Default value: "default"
|
||||
**See Also**
|
||||
- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
|
||||
|
||||
## workload_path {#workload_path}
|
||||
|
||||
The directory used as a storage for all `CREATE WORKLOAD` and `CREATE RESOURCE` queries. By default `/workload/` folder under server working directory is used.
|
||||
|
||||
**Example**
|
||||
|
||||
``` xml
|
||||
<workload_path>/var/lib/clickhouse/workload/</workload_path>
|
||||
```
|
||||
|
||||
**See Also**
|
||||
- [Workload Hierarchy](/docs/en/operations/workload-scheduling.md#workloads)
|
||||
- [workload_zookeeper_path](#workload_zookeeper_path)
|
||||
|
||||
## workload_zookeeper_path {#workload_zookeeper_path}
|
||||
|
||||
The path to a ZooKeeper node, which is used as a storage for all `CREATE WORKLOAD` and `CREATE RESOURCE` queries. For consistency all SQL definitions are stored as a value of this single znode. By default ZooKeeper is not used and definitions are stored on [disk](#workload_path).
|
||||
|
||||
**Example**
|
||||
|
||||
``` xml
|
||||
<workload_zookeeper_path>/clickhouse/workload/definitions.sql</workload_zookeeper_path>
|
||||
```
|
||||
|
||||
**See Also**
|
||||
- [Workload Hierarchy](/docs/en/operations/workload-scheduling.md#workloads)
|
||||
- [workload_path](#workload_path)
|
||||
|
||||
## max_authentication_methods_per_user {#max_authentication_methods_per_user}
|
||||
|
||||
The maximum number of authentication methods a user can be created with or altered to.
|
||||
|
||||
@ -78,6 +78,16 @@ If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled.
|
||||
|
||||
Default value: `10 * 1024 * 1024 * 1024` bytes.
|
||||
|
||||
## ttl_only_drop_parts
|
||||
|
||||
Controls whether data parts are fully dropped in MergeTree tables when all rows in that part have expired according to their `TTL` settings.
|
||||
|
||||
When `ttl_only_drop_parts` is disabled (by default), only the rows that have expired based on their TTL settings are removed.
|
||||
|
||||
When `ttl_only_drop_parts` is enabled, the entire part is dropped if all rows in that part have expired according to their `TTL` settings.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## merge_with_ttl_timeout
|
||||
|
||||
Minimum delay in seconds before repeating a merge with delete TTL.
|
||||
@ -1095,3 +1105,13 @@ Possible values:
|
||||
Default value: 0.0
|
||||
|
||||
Note that if both `min_free_disk_ratio_to_perform_insert` and `min_free_disk_bytes_to_perform_insert` are specified, ClickHouse will count on the value that will allow to perform inserts on a bigger amount of free memory.
|
||||
|
||||
## cache_populated_by_fetch
|
||||
|
||||
A Cloud only setting.
|
||||
|
||||
When `cache_populated_by_fetch` is disabled (the default setting), new data parts are loaded into the cache only when a query is run that requires those parts.
|
||||
|
||||
If enabled, `cache_populated_by_fetch` will instead cause all nodes to load new data parts from storage into their cache without requiring a query to trigger such an action.
|
||||
|
||||
Default value: 0.
|
||||
@ -211,7 +211,7 @@ Number of threads in the server of the replicas communication protocol (without
|
||||
|
||||
The difference in time the thread for calculation of the asynchronous metrics was scheduled to wake up and the time it was in fact, woken up. A proxy-indicator of overall system latency and responsiveness.
|
||||
|
||||
### LoadAverage_*N*
|
||||
### LoadAverage*N*
|
||||
|
||||
The whole system load, averaged with exponential smoothing over 1 minute. The load represents the number of threads across all the processes (the scheduling entities of the OS kernel), that are currently running by CPU or waiting for IO, or ready to run but not being scheduled at this point of time. This number includes all the processes, not only clickhouse-server. The number can be greater than the number of CPU cores, if the system is overloaded, and many processes are ready to run but waiting for CPU or IO.
|
||||
|
||||
|
||||
@ -19,7 +19,7 @@ Columns:
|
||||
- `column` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Name of a column to which access is granted.
|
||||
|
||||
- `is_partial_revoke` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Logical value. It shows whether some privileges have been revoked. Possible values:
|
||||
- `0` — The row describes a partial revoke.
|
||||
- `1` — The row describes a grant.
|
||||
- `0` — The row describes a grant.
|
||||
- `1` — The row describes a partial revoke.
|
||||
|
||||
- `grant_option` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Permission is granted `WITH GRANT OPTION`, see [GRANT](../../sql-reference/statements/grant.md#granting-privilege-syntax).
|
||||
|
||||
@ -18,6 +18,11 @@ Columns:
|
||||
- `1` — Current user can’t change the setting.
|
||||
- `type` ([String](../../sql-reference/data-types/string.md)) — Setting type (implementation specific string value).
|
||||
- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) - Shows whether a setting is obsolete.
|
||||
- `tier` ([Enum8](../../sql-reference/data-types/enum.md)) — Support level for this feature. ClickHouse features are organized in tiers, varying depending on the current status of their development and the expectations one might have when using them. Values:
|
||||
- `'Production'` — The feature is stable, safe to use and does not have issues interacting with other **production** features. .
|
||||
- `'Beta'` — The feature is stable and safe. The outcome of using it together with other features is unknown and correctness is not guaranteed. Testing and reports are welcome.
|
||||
- `'Experimental'` — The feature is under development. Only intended for developers and ClickHouse enthusiasts. The feature might or might not work and could be removed at any time.
|
||||
- `'Obsolete'` — No longer supported. Either it is already removed or it will be removed in future releases.
|
||||
|
||||
**Example**
|
||||
```sql
|
||||
|
||||
37
docs/en/operations/system-tables/resources.md
Normal file
37
docs/en/operations/system-tables/resources.md
Normal file
@ -0,0 +1,37 @@
|
||||
---
|
||||
slug: /en/operations/system-tables/resources
|
||||
---
|
||||
# resources
|
||||
|
||||
Contains information for [resources](/docs/en/operations/workload-scheduling.md#workload_entity_storage) residing on the local server. The table contains a row for every resource.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT *
|
||||
FROM system.resources
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
name: io_read
|
||||
read_disks: ['s3']
|
||||
write_disks: []
|
||||
create_query: CREATE RESOURCE io_read (READ DISK s3)
|
||||
|
||||
Row 2:
|
||||
──────
|
||||
name: io_write
|
||||
read_disks: []
|
||||
write_disks: ['s3']
|
||||
create_query: CREATE RESOURCE io_write (WRITE DISK s3)
|
||||
```
|
||||
|
||||
Columns:
|
||||
|
||||
- `name` (`String`) - Resource name.
|
||||
- `read_disks` (`Array(String)`) - The array of disk names that uses this resource for read operations.
|
||||
- `write_disks` (`Array(String)`) - The array of disk names that uses this resource for write operations.
|
||||
- `create_query` (`String`) - The definition of the resource.
|
||||
@ -18,6 +18,11 @@ Columns:
|
||||
- `1` — Current user can’t change the setting.
|
||||
- `default` ([String](../../sql-reference/data-types/string.md)) — Setting default value.
|
||||
- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) - Shows whether a setting is obsolete.
|
||||
- `tier` ([Enum8](../../sql-reference/data-types/enum.md)) — Support level for this feature. ClickHouse features are organized in tiers, varying depending on the current status of their development and the expectations one might have when using them. Values:
|
||||
- `'Production'` — The feature is stable, safe to use and does not have issues interacting with other **production** features. .
|
||||
- `'Beta'` — The feature is stable and safe. The outcome of using it together with other features is unknown and correctness is not guaranteed. Testing and reports are welcome.
|
||||
- `'Experimental'` — The feature is under development. Only intended for developers and ClickHouse enthusiasts. The feature might or might not work and could be removed at any time.
|
||||
- `'Obsolete'` — No longer supported. Either it is already removed or it will be removed in future releases.
|
||||
|
||||
**Example**
|
||||
|
||||
@ -26,19 +31,99 @@ The following example shows how to get information about settings which name con
|
||||
``` sql
|
||||
SELECT *
|
||||
FROM system.settings
|
||||
WHERE name LIKE '%min_i%'
|
||||
WHERE name LIKE '%min_insert_block_size_%'
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─name───────────────────────────────────────────────_─value─────_─changed─_─description───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────_─min──_─max──_─readonly─_─type─────────_─default───_─alias_for─_─is_obsolete─┐
|
||||
│ min_insert_block_size_rows │ 1048449 │ 0 │ Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough. │ ____ │ ____ │ 0 │ UInt64 │ 1048449 │ │ 0 │
|
||||
│ min_insert_block_size_bytes │ 268402944 │ 0 │ Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough. │ ____ │ ____ │ 0 │ UInt64 │ 268402944 │ │ 0 │
|
||||
│ min_insert_block_size_rows_for_materialized_views │ 0 │ 0 │ Like min_insert_block_size_rows, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_rows) │ ____ │ ____ │ 0 │ UInt64 │ 0 │ │ 0 │
|
||||
│ min_insert_block_size_bytes_for_materialized_views │ 0 │ 0 │ Like min_insert_block_size_bytes, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_bytes) │ ____ │ ____ │ 0 │ UInt64 │ 0 │ │ 0 │
|
||||
│ read_backoff_min_interval_between_events_ms │ 1000 │ 0 │ Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time. │ ____ │ ____ │ 0 │ Milliseconds │ 1000 │ │ 0 │
|
||||
└────────────────────────────────────────────────────┴───────────┴─────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
──────────────────────────────────────────────────────┴──────┴──────┴──────────┴──────────────┴───────────┴───────────┴─────────────┘
|
||||
```
|
||||
Row 1:
|
||||
──────
|
||||
name: min_insert_block_size_rows
|
||||
value: 1048449
|
||||
changed: 0
|
||||
description: Sets the minimum number of rows in the block that can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Squashing disabled.
|
||||
min: ᴺᵁᴸᴸ
|
||||
max: ᴺᵁᴸᴸ
|
||||
readonly: 0
|
||||
type: UInt64
|
||||
default: 1048449
|
||||
alias_for:
|
||||
is_obsolete: 0
|
||||
tier: Production
|
||||
|
||||
Row 2:
|
||||
──────
|
||||
name: min_insert_block_size_bytes
|
||||
value: 268402944
|
||||
changed: 0
|
||||
description: Sets the minimum number of bytes in the block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Squashing disabled.
|
||||
min: ᴺᵁᴸᴸ
|
||||
max: ᴺᵁᴸᴸ
|
||||
readonly: 0
|
||||
type: UInt64
|
||||
default: 268402944
|
||||
alias_for:
|
||||
is_obsolete: 0
|
||||
tier: Production
|
||||
|
||||
Row 3:
|
||||
──────
|
||||
name: min_insert_block_size_rows_for_materialized_views
|
||||
value: 0
|
||||
changed: 0
|
||||
description: Sets the minimum number of rows in the block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. This setting is applied only for blocks inserted into [materialized view](../../sql-reference/statements/create/view.md). By adjusting this setting, you control blocks squashing while pushing to materialized view and avoid excessive memory usage.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
- 0 — Squashing disabled.
|
||||
|
||||
**See Also**
|
||||
|
||||
- [min_insert_block_size_rows](#min-insert-block-size-rows)
|
||||
min: ᴺᵁᴸᴸ
|
||||
max: ᴺᵁᴸᴸ
|
||||
readonly: 0
|
||||
type: UInt64
|
||||
default: 0
|
||||
alias_for:
|
||||
is_obsolete: 0
|
||||
tier: Production
|
||||
|
||||
Row 4:
|
||||
──────
|
||||
name: min_insert_block_size_bytes_for_materialized_views
|
||||
value: 0
|
||||
changed: 0
|
||||
description: Sets the minimum number of bytes in the block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. This setting is applied only for blocks inserted into [materialized view](../../sql-reference/statements/create/view.md). By adjusting this setting, you control blocks squashing while pushing to materialized view and avoid excessive memory usage.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
- 0 — Squashing disabled.
|
||||
|
||||
**See also**
|
||||
|
||||
- [min_insert_block_size_bytes](#min-insert-block-size-bytes)
|
||||
min: ᴺᵁᴸᴸ
|
||||
max: ᴺᵁᴸᴸ
|
||||
readonly: 0
|
||||
type: UInt64
|
||||
default: 0
|
||||
alias_for:
|
||||
is_obsolete: 0
|
||||
tier: Production
|
||||
```
|
||||
|
||||
Using of `WHERE changed` can be useful, for example, when you want to check:
|
||||
|
||||
|
||||
40
docs/en/operations/system-tables/workloads.md
Normal file
40
docs/en/operations/system-tables/workloads.md
Normal file
@ -0,0 +1,40 @@
|
||||
---
|
||||
slug: /en/operations/system-tables/workloads
|
||||
---
|
||||
# workloads
|
||||
|
||||
Contains information for [workloads](/docs/en/operations/workload-scheduling.md#workload_entity_storage) residing on the local server. The table contains a row for every workload.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT *
|
||||
FROM system.workloads
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
name: production
|
||||
parent: all
|
||||
create_query: CREATE WORKLOAD production IN `all` SETTINGS weight = 9
|
||||
|
||||
Row 2:
|
||||
──────
|
||||
name: development
|
||||
parent: all
|
||||
create_query: CREATE WORKLOAD development IN `all`
|
||||
|
||||
Row 3:
|
||||
──────
|
||||
name: all
|
||||
parent:
|
||||
create_query: CREATE WORKLOAD `all`
|
||||
```
|
||||
|
||||
Columns:
|
||||
|
||||
- `name` (`String`) - Workload name.
|
||||
- `parent` (`String`) - Parent workload name.
|
||||
- `create_query` (`String`) - The definition of the workload.
|
||||
@ -43,6 +43,20 @@ Example:
|
||||
</clickhouse>
|
||||
```
|
||||
|
||||
An alternative way to express which disks are used by a resource is SQL syntax:
|
||||
|
||||
```sql
|
||||
CREATE RESOURCE resource_name (WRITE DISK disk1, READ DISK disk2)
|
||||
```
|
||||
|
||||
Resource could be used for any number of disk for READ or WRITE or both for READ and WRITE. There a syntax allowing to use a resource for all the disks:
|
||||
|
||||
```sql
|
||||
CREATE RESOURCE all_io (READ ANY DISK, WRITE ANY DISK);
|
||||
```
|
||||
|
||||
Note that server configuration options have priority over SQL way to define resources.
|
||||
|
||||
## Workload markup {#workload_markup}
|
||||
|
||||
Queries can be marked with setting `workload` to distinguish different workloads. If `workload` is not set, than value "default" is used. Note that you are able to specify the other value using settings profiles. Setting constraints can be used to make `workload` constant if you want all queries from the user to be marked with fixed value of `workload` setting.
|
||||
@ -153,9 +167,48 @@ Example:
|
||||
</clickhouse>
|
||||
```
|
||||
|
||||
## Workload hierarchy (SQL only) {#workloads}
|
||||
|
||||
Defining resources and classifiers in XML could be challenging. ClickHouse provides SQL syntax that is much more convenient. All resources that were created with `CREATE RESOURCE` share the same structure of the hierarchy, but could differ in some aspects. Every workload created with `CREATE WORKLOAD` maintains a few automatically created scheduling nodes for every resource. A child workload can be created inside another parent workload. Here is the example that defines exactly the same hierarchy as XML configuration above:
|
||||
|
||||
```sql
|
||||
CREATE RESOURCE network_write (WRITE DISK s3)
|
||||
CREATE RESOURCE network_read (READ DISK s3)
|
||||
CREATE WORKLOAD all SETTINGS max_requests = 100
|
||||
CREATE WORKLOAD development IN all
|
||||
CREATE WORKLOAD production IN all SETTINGS weight = 3
|
||||
```
|
||||
|
||||
The name of a leaf workload without children could be used in query settings `SETTINGS workload = 'name'`. Note that workload classifiers are also created automatically when using SQL syntax.
|
||||
|
||||
To customize workload the following settings could be used:
|
||||
* `priority` - sibling workloads are served according to static priority values (lower value means higher priority).
|
||||
* `weight` - sibling workloads having the same static priority share resources according to weights.
|
||||
* `max_requests` - the limit on the number of concurrent resource requests in this workload.
|
||||
* `max_cost` - the limit on the total inflight bytes count of concurrent resource requests in this workload.
|
||||
* `max_speed` - the limit on byte processing rate of this workload (the limit is independent for every resource).
|
||||
* `max_burst` - maximum number of bytes that could be processed by the workload without being throttled (for every resource independently).
|
||||
|
||||
Note that workload settings are translated into a proper set of scheduling nodes. For more details, see the description of the scheduling node [types and options](#hierarchy).
|
||||
|
||||
There is no way to specify different hierarchies of workloads for different resources. But there is a way to specify different workload setting value for a specific resource:
|
||||
|
||||
```sql
|
||||
CREATE OR REPLACE WORKLOAD all SETTINGS max_requests = 100, max_speed = 1000000 FOR network_read, max_speed = 2000000 FOR network_write
|
||||
```
|
||||
|
||||
Also note that workload or resource could not be dropped if it is referenced from another workload. To update a definition of a workload use `CREATE OR REPLACE WORKLOAD` query.
|
||||
|
||||
## Workloads and resources storage {#workload_entity_storage}
|
||||
Definitions of all workloads and resources in the form of `CREATE WORKLOAD` and `CREATE RESOURCE` queries are stored persistently either on disk at `workload_path` or in ZooKeeper at `workload_zookeeper_path`. ZooKeeper storage is recommended to achieve consistency between nodes. Alternatively `ON CLUSTER` clause could be used along with disk storage.
|
||||
|
||||
## See also
|
||||
- [system.scheduler](/docs/en/operations/system-tables/scheduler.md)
|
||||
- [system.workloads](/docs/en/operations/system-tables/workloads.md)
|
||||
- [system.resources](/docs/en/operations/system-tables/resources.md)
|
||||
- [merge_workload](/docs/en/operations/settings/merge-tree-settings.md#merge_workload) merge tree setting
|
||||
- [merge_workload](/docs/en/operations/server-configuration-parameters/settings.md#merge_workload) global server setting
|
||||
- [mutation_workload](/docs/en/operations/settings/merge-tree-settings.md#mutation_workload) merge tree setting
|
||||
- [mutation_workload](/docs/en/operations/server-configuration-parameters/settings.md#mutation_workload) global server setting
|
||||
- [workload_path](/docs/en/operations/server-configuration-parameters/settings.md#workload_path) global server setting
|
||||
- [workload_zookeeper_path](/docs/en/operations/server-configuration-parameters/settings.md#workload_zookeeper_path) global server setting
|
||||
|
||||
@ -75,7 +75,7 @@ FROM t_null_big
|
||||
└────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
Also you can use [Tuple](/docs/en/sql-reference/data-types/tuple.md) to work around NULL skipping behavior. The a `Tuple` that contains only a `NULL` value is not `NULL`, so the aggregate functions won't skip that row because of that `NULL` value.
|
||||
Also you can use [Tuple](/docs/en/sql-reference/data-types/tuple.md) to work around NULL skipping behavior. A `Tuple` that contains only a `NULL` value is not `NULL`, so the aggregate functions won't skip that row because of that `NULL` value.
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
@ -110,7 +110,7 @@ GROUP BY v
|
||||
└──────┴─────────┴──────────┘
|
||||
```
|
||||
|
||||
And here is an example of of first_value with `RESPECT NULLS` where we can see that NULL inputs are respected and it will return the first value read, whether it's NULL or not:
|
||||
And here is an example of first_value with `RESPECT NULLS` where we can see that NULL inputs are respected and it will return the first value read, whether it's NULL or not:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
|
||||
@ -5,7 +5,15 @@ sidebar_position: 102
|
||||
|
||||
# any
|
||||
|
||||
Selects the first encountered value of a column, ignoring any `NULL` values.
|
||||
Selects the first encountered value of a column.
|
||||
|
||||
:::warning
|
||||
As a query can be executed in arbitrary order, the result of this function is non-deterministic.
|
||||
If you need an arbitrary but deterministic result, use functions [`min`](../reference/min.md) or [`max`](../reference/max.md).
|
||||
:::
|
||||
|
||||
By default, the function never returns NULL, i.e. ignores NULL values in the input column.
|
||||
However, if the function is used with the `RESPECT NULLS` modifier, it returns the first value reads no matter if NULL or not.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -13,46 +21,51 @@ Selects the first encountered value of a column, ignoring any `NULL` values.
|
||||
any(column) [RESPECT NULLS]
|
||||
```
|
||||
|
||||
Aliases: `any_value`, [`first_value`](../reference/first_value.md).
|
||||
Aliases `any(column)` (without `RESPECT NULLS`)
|
||||
- `any_value`
|
||||
- [`first_value`](../reference/first_value.md).
|
||||
|
||||
Alias for `any(column) RESPECT NULLS`
|
||||
- `anyRespectNulls`, `any_respect_nulls`
|
||||
- `firstValueRespectNulls`, `first_value_respect_nulls`
|
||||
- `anyValueRespectNulls`, `any_value_respect_nulls`
|
||||
|
||||
**Parameters**
|
||||
- `column`: The column name.
|
||||
|
||||
**Returned value**
|
||||
|
||||
:::note
|
||||
Supports the `RESPECT NULLS` modifier after the function name. Using this modifier will ensure the function selects the first value passed, regardless of whether it is `NULL` or not.
|
||||
:::
|
||||
The first value encountered.
|
||||
|
||||
:::note
|
||||
The return type of the function is the same as the input, except for LowCardinality which is discarded. This means that given no rows as input it will return the default value of that type (0 for integers, or Null for a Nullable() column). You might use the `-OrNull` [combinator](../../../sql-reference/aggregate-functions/combinators.md) ) to modify this behaviour.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
The query can be executed in any order and even in a different order each time, so the result of this function is indeterminate.
|
||||
To get a determinate result, you can use the [`min`](../reference/min.md) or [`max`](../reference/max.md) function instead of `any`.
|
||||
The return type of the function is the same as the input, except for LowCardinality which is discarded.
|
||||
This means that given no rows as input it will return the default value of that type (0 for integers, or Null for a Nullable() column).
|
||||
You might use the `-OrNull` [combinator](../../../sql-reference/aggregate-functions/combinators.md) ) to modify this behaviour.
|
||||
:::
|
||||
|
||||
**Implementation details**
|
||||
|
||||
In some cases, you can rely on the order of execution. This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`.
|
||||
In some cases, you can rely on the order of execution.
|
||||
This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`.
|
||||
|
||||
When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
|
||||
When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions.
|
||||
In other words, each column selected from the table must be used either in keys or inside aggregate functions.
|
||||
To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
CREATE TABLE any_nulls (city Nullable(String)) ENGINE=Log;
|
||||
CREATE TABLE tab (city Nullable(String)) ENGINE=Memory;
|
||||
|
||||
INSERT INTO any_nulls (city) VALUES (NULL), ('Amsterdam'), ('New York'), ('Tokyo'), ('Valencia'), (NULL);
|
||||
INSERT INTO tab (city) VALUES (NULL), ('Amsterdam'), ('New York'), ('Tokyo'), ('Valencia'), (NULL);
|
||||
|
||||
SELECT any(city) FROM any_nulls;
|
||||
SELECT any(city), anyRespectNulls(city) FROM tab;
|
||||
```
|
||||
|
||||
```response
|
||||
┌─any(city)─┐
|
||||
│ Amsterdam │
|
||||
└───────────┘
|
||||
┌─any(city)─┬─anyRespectNulls(city)─┐
|
||||
│ Amsterdam │ ᴺᵁᴸᴸ │
|
||||
└───────────┴───────────────────────┘
|
||||
```
|
||||
|
||||
@ -5,7 +5,15 @@ sidebar_position: 105
|
||||
|
||||
# anyLast
|
||||
|
||||
Selects the last value encountered, ignoring any `NULL` values by default. The result is just as indeterminate as for the [any](../../../sql-reference/aggregate-functions/reference/any.md) function.
|
||||
Selects the last encountered value of a column.
|
||||
|
||||
:::warning
|
||||
As a query can be executed in arbitrary order, the result of this function is non-deterministic.
|
||||
If you need an arbitrary but deterministic result, use functions [`min`](../reference/min.md) or [`max`](../reference/max.md).
|
||||
:::
|
||||
|
||||
By default, the function never returns NULL, i.e. ignores NULL values in the input column.
|
||||
However, if the function is used with the `RESPECT NULLS` modifier, it returns the first value reads no matter if NULL or not.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -13,13 +21,16 @@ Selects the last value encountered, ignoring any `NULL` values by default. The r
|
||||
anyLast(column) [RESPECT NULLS]
|
||||
```
|
||||
|
||||
Alias `anyLast(column)` (without `RESPECT NULLS`)
|
||||
- [`last_value`](../reference/last_value.md).
|
||||
|
||||
Aliases for `anyLast(column) RESPECT NULLS`
|
||||
- `anyLastRespectNulls`, `anyLast_respect_nulls`
|
||||
- `lastValueRespectNulls`, `last_value_respect_nulls`
|
||||
|
||||
**Parameters**
|
||||
- `column`: The column name.
|
||||
|
||||
:::note
|
||||
Supports the `RESPECT NULLS` modifier after the function name. Using this modifier will ensure the function selects the first value passed, regardless of whether it is `NULL` or not.
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The last value encountered.
|
||||
@ -29,15 +40,15 @@ Supports the `RESPECT NULLS` modifier after the function name. Using this modifi
|
||||
Query:
|
||||
|
||||
```sql
|
||||
CREATE TABLE any_last_nulls (city Nullable(String)) ENGINE=Log;
|
||||
CREATE TABLE tab (city Nullable(String)) ENGINE=Memory;
|
||||
|
||||
INSERT INTO any_last_nulls (city) VALUES ('Amsterdam'),(NULL),('New York'),('Tokyo'),('Valencia'),(NULL);
|
||||
INSERT INTO tab (city) VALUES ('Amsterdam'),(NULL),('New York'),('Tokyo'),('Valencia'),(NULL);
|
||||
|
||||
SELECT anyLast(city) FROM any_last_nulls;
|
||||
SELECT anyLast(city), anyLastRespectNulls(city) FROM tab;
|
||||
```
|
||||
|
||||
```response
|
||||
┌─anyLast(city)─┐
|
||||
│ Valencia │
|
||||
└───────────────┘
|
||||
┌─anyLast(city)─┬─anyLastRespectNulls(city)─┐
|
||||
│ Valencia │ ᴺᵁᴸᴸ │
|
||||
└───────────────┴───────────────────────────┘
|
||||
```
|
||||
@ -7,119 +7,4 @@ toc_hidden: true
|
||||
|
||||
# List of Aggregate Functions
|
||||
|
||||
Standard aggregate functions:
|
||||
|
||||
- [count](../reference/count.md)
|
||||
- [min](../reference/min.md)
|
||||
- [max](../reference/max.md)
|
||||
- [sum](../reference/sum.md)
|
||||
- [avg](../reference/avg.md)
|
||||
- [any](../reference/any.md)
|
||||
- [stddevPop](../reference/stddevpop.md)
|
||||
- [stddevPopStable](../reference/stddevpopstable.md)
|
||||
- [stddevSamp](../reference/stddevsamp.md)
|
||||
- [stddevSampStable](../reference/stddevsampstable.md)
|
||||
- [varPop](../reference/varpop.md)
|
||||
- [varSamp](../reference/varsamp.md)
|
||||
- [corr](../reference/corr.md)
|
||||
- [corr](../reference/corrstable.md)
|
||||
- [corrMatrix](../reference/corrmatrix.md)
|
||||
- [covarPop](../reference/covarpop.md)
|
||||
- [covarStable](../reference/covarpopstable.md)
|
||||
- [covarPopMatrix](../reference/covarpopmatrix.md)
|
||||
- [covarSamp](../reference/covarsamp.md)
|
||||
- [covarSampStable](../reference/covarsampstable.md)
|
||||
- [covarSampMatrix](../reference/covarsampmatrix.md)
|
||||
- [entropy](../reference/entropy.md)
|
||||
- [exponentialMovingAverage](../reference/exponentialmovingaverage.md)
|
||||
- [intervalLengthSum](../reference/intervalLengthSum.md)
|
||||
- [kolmogorovSmirnovTest](../reference/kolmogorovsmirnovtest.md)
|
||||
- [mannwhitneyutest](../reference/mannwhitneyutest.md)
|
||||
- [median](../reference/median.md)
|
||||
- [rankCorr](../reference/rankCorr.md)
|
||||
- [sumKahan](../reference/sumkahan.md)
|
||||
- [studentTTest](../reference/studentttest.md)
|
||||
- [welchTTest](../reference/welchttest.md)
|
||||
|
||||
ClickHouse-specific aggregate functions:
|
||||
|
||||
- [aggThrow](../reference/aggthrow.md)
|
||||
- [analysisOfVariance](../reference/analysis_of_variance.md)
|
||||
- [any](../reference/any.md)
|
||||
- [anyHeavy](../reference/anyheavy.md)
|
||||
- [anyLast](../reference/anylast.md)
|
||||
- [boundingRatio](../reference/boundrat.md)
|
||||
- [first_value](../reference/first_value.md)
|
||||
- [last_value](../reference/last_value.md)
|
||||
- [argMin](../reference/argmin.md)
|
||||
- [argMax](../reference/argmax.md)
|
||||
- [avgWeighted](../reference/avgweighted.md)
|
||||
- [topK](../reference/topk.md)
|
||||
- [topKWeighted](../reference/topkweighted.md)
|
||||
- [deltaSum](../reference/deltasum.md)
|
||||
- [deltaSumTimestamp](../reference/deltasumtimestamp.md)
|
||||
- [flameGraph](../reference/flame_graph.md)
|
||||
- [groupArray](../reference/grouparray.md)
|
||||
- [groupArrayLast](../reference/grouparraylast.md)
|
||||
- [groupUniqArray](../reference/groupuniqarray.md)
|
||||
- [groupArrayInsertAt](../reference/grouparrayinsertat.md)
|
||||
- [groupArrayMovingAvg](../reference/grouparraymovingavg.md)
|
||||
- [groupArrayMovingSum](../reference/grouparraymovingsum.md)
|
||||
- [groupArraySample](../reference/grouparraysample.md)
|
||||
- [groupArraySorted](../reference/grouparraysorted.md)
|
||||
- [groupArrayIntersect](../reference/grouparrayintersect.md)
|
||||
- [groupBitAnd](../reference/groupbitand.md)
|
||||
- [groupBitOr](../reference/groupbitor.md)
|
||||
- [groupBitXor](../reference/groupbitxor.md)
|
||||
- [groupBitmap](../reference/groupbitmap.md)
|
||||
- [groupBitmapAnd](../reference/groupbitmapand.md)
|
||||
- [groupBitmapOr](../reference/groupbitmapor.md)
|
||||
- [groupBitmapXor](../reference/groupbitmapxor.md)
|
||||
- [sumWithOverflow](../reference/sumwithoverflow.md)
|
||||
- [sumMap](../reference/summap.md)
|
||||
- [sumMapWithOverflow](../reference/summapwithoverflow.md)
|
||||
- [sumMapFiltered](../parametric-functions.md/#summapfiltered)
|
||||
- [sumMapFilteredWithOverflow](../parametric-functions.md/#summapfilteredwithoverflow)
|
||||
- [minMap](../reference/minmap.md)
|
||||
- [maxMap](../reference/maxmap.md)
|
||||
- [skewSamp](../reference/skewsamp.md)
|
||||
- [skewPop](../reference/skewpop.md)
|
||||
- [kurtSamp](../reference/kurtsamp.md)
|
||||
- [kurtPop](../reference/kurtpop.md)
|
||||
- [uniq](../reference/uniq.md)
|
||||
- [uniqExact](../reference/uniqexact.md)
|
||||
- [uniqCombined](../reference/uniqcombined.md)
|
||||
- [uniqCombined64](../reference/uniqcombined64.md)
|
||||
- [uniqHLL12](../reference/uniqhll12.md)
|
||||
- [uniqTheta](../reference/uniqthetasketch.md)
|
||||
- [quantile](../reference/quantile.md)
|
||||
- [quantiles](../reference/quantiles.md)
|
||||
- [quantileExact](../reference/quantileexact.md)
|
||||
- [quantileExactLow](../reference/quantileexact.md#quantileexactlow)
|
||||
- [quantileExactHigh](../reference/quantileexact.md#quantileexacthigh)
|
||||
- [quantileExactWeighted](../reference/quantileexactweighted.md)
|
||||
- [quantileTiming](../reference/quantiletiming.md)
|
||||
- [quantileTimingWeighted](../reference/quantiletimingweighted.md)
|
||||
- [quantileDeterministic](../reference/quantiledeterministic.md)
|
||||
- [quantileTDigest](../reference/quantiletdigest.md)
|
||||
- [quantileTDigestWeighted](../reference/quantiletdigestweighted.md)
|
||||
- [quantileBFloat16](../reference/quantilebfloat16.md#quantilebfloat16)
|
||||
- [quantileBFloat16Weighted](../reference/quantilebfloat16.md#quantilebfloat16weighted)
|
||||
- [quantileDD](../reference/quantileddsketch.md#quantileddsketch)
|
||||
- [simpleLinearRegression](../reference/simplelinearregression.md)
|
||||
- [singleValueOrNull](../reference/singlevalueornull.md)
|
||||
- [stochasticLinearRegression](../reference/stochasticlinearregression.md)
|
||||
- [stochasticLogisticRegression](../reference/stochasticlogisticregression.md)
|
||||
- [categoricalInformationValue](../reference/categoricalinformationvalue.md)
|
||||
- [contingency](../reference/contingency.md)
|
||||
- [cramersV](../reference/cramersv.md)
|
||||
- [cramersVBiasCorrected](../reference/cramersvbiascorrected.md)
|
||||
- [theilsU](../reference/theilsu.md)
|
||||
- [maxIntersections](../reference/maxintersections.md)
|
||||
- [maxIntersectionsPosition](../reference/maxintersectionsposition.md)
|
||||
- [meanZTest](../reference/meanztest.md)
|
||||
- [quantileGK](../reference/quantileGK.md)
|
||||
- [quantileInterpolatedWeighted](../reference/quantileinterpolatedweighted.md)
|
||||
- [sparkBar](../reference/sparkbar.md)
|
||||
- [sumCount](../reference/sumcount.md)
|
||||
- [largestTriangleThreeBuckets](../reference/largestTriangleThreeBuckets.md)
|
||||
ClickHouse supports all standard SQL aggregate functions ([sum](../reference/sum.md), [avg](../reference/avg.md), [min](../reference/min.md), [max](../reference/max.md), [count](../reference/count.md)), as well as a wide range of other aggregate functions.
|
||||
|
||||
@ -6,7 +6,9 @@ sidebar_label: AggregateFunction
|
||||
|
||||
# AggregateFunction
|
||||
|
||||
Aggregate functions can have an implementation-defined intermediate state that can be serialized to an `AggregateFunction(...)` data type and stored in a table, usually, by means of [a materialized view](../../sql-reference/statements/create/view.md). The common way to produce an aggregate function state is by calling the aggregate function with the `-State` suffix. To get the final result of aggregation in the future, you must use the same aggregate function with the `-Merge`suffix.
|
||||
Aggregate functions have an implementation-defined intermediate state that can be serialized to an `AggregateFunction(...)` data type and stored in a table, usually, by means of [a materialized view](../../sql-reference/statements/create/view.md).
|
||||
The common way to produce an aggregate function state is by calling the aggregate function with the `-State` suffix.
|
||||
To get the final result of aggregation in the future, you must use the same aggregate function with the `-Merge`suffix.
|
||||
|
||||
`AggregateFunction(name, types_of_arguments...)` — parametric data type.
|
||||
|
||||
|
||||
@ -512,6 +512,8 @@ The result of operator `<` for values `d1` with underlying type `T1` and `d2` wi
|
||||
- If `T1 = T2 = T`, the result will be `d1.T < d2.T` (underlying values will be compared).
|
||||
- If `T1 != T2`, the result will be `T1 < T2` (type names will be compared).
|
||||
|
||||
By default `Dynamic` type is not allowed in `GROUP BY`/`ORDER BY` keys, if you want to use it consider its special comparison rule and enable `allow_suspicious_types_in_group_by`/`allow_suspicious_types_in_order_by` settings.
|
||||
|
||||
Examples:
|
||||
```sql
|
||||
CREATE TABLE test (d Dynamic) ENGINE=Memory;
|
||||
@ -535,7 +537,7 @@ SELECT d, dynamicType(d) FROM test;
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, dynamicType(d) FROM test ORDER BY d;
|
||||
SELECT d, dynamicType(d) FROM test ORDER BY d SETTINGS allow_suspicious_types_in_order_by=1;
|
||||
```
|
||||
|
||||
```sql
|
||||
@ -557,7 +559,7 @@ Example:
|
||||
```sql
|
||||
CREATE TABLE test (d Dynamic) ENGINE=Memory;
|
||||
INSERT INTO test VALUES (1::UInt32), (1::Int64), (100::UInt32), (100::Int64);
|
||||
SELECT d, dynamicType(d) FROM test ORDER by d;
|
||||
SELECT d, dynamicType(d) FROM test ORDER BY d SETTINGS allow_suspicious_types_in_order_by=1;
|
||||
```
|
||||
|
||||
```text
|
||||
@ -570,7 +572,7 @@ SELECT d, dynamicType(d) FROM test ORDER by d;
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, dynamicType(d) FROM test GROUP by d;
|
||||
SELECT d, dynamicType(d) FROM test GROUP by d SETTINGS allow_suspicious_types_in_group_by=1;
|
||||
```
|
||||
|
||||
```text
|
||||
@ -582,7 +584,7 @@ SELECT d, dynamicType(d) FROM test GROUP by d;
|
||||
└─────┴────────────────┘
|
||||
```
|
||||
|
||||
**Note**: the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type
|
||||
**Note:** the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type
|
||||
|
||||
## Reaching the limit in number of different data types stored inside Dynamic
|
||||
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/float
|
||||
sidebar_position: 4
|
||||
sidebar_label: Float32, Float64
|
||||
sidebar_label: Float32, Float64, BFloat16
|
||||
---
|
||||
|
||||
# Float32, Float64
|
||||
# Float32, Float64, BFloat16
|
||||
|
||||
:::note
|
||||
If you need accurate calculations, in particular if you work with financial or business data requiring a high precision, you should consider using [Decimal](../data-types/decimal.md) instead.
|
||||
@ -117,3 +117,11 @@ SELECT 0 / 0
|
||||
```
|
||||
|
||||
See the rules for `NaN` sorting in the section [ORDER BY clause](../../sql-reference/statements/select/order-by.md).
|
||||
|
||||
## BFloat16
|
||||
|
||||
`BFloat16` is a 16-bit floating point data type with 8-bit exponent, sign, and 7-bit mantissa.
|
||||
|
||||
It is useful for machine learning and AI applications.
|
||||
|
||||
ClickHouse supports conversions between `Float32` and `BFloat16`. Most of other operations are not supported.
|
||||
|
||||
@ -6,29 +6,8 @@ sidebar_position: 1
|
||||
|
||||
# Data Types in ClickHouse
|
||||
|
||||
ClickHouse can store various kinds of data in table cells. This section describes the supported data types and special considerations for using and/or implementing them if any.
|
||||
This section describes the data types supported by ClickHouse, for example [integers](int-uint.md), [floats](float.md) and [strings](string.md).
|
||||
|
||||
:::note
|
||||
You can check whether a data type name is case-sensitive in the [system.data_type_families](../../operations/system-tables/data_type_families.md#system_tables-data_type_families) table.
|
||||
:::
|
||||
|
||||
ClickHouse data types include:
|
||||
|
||||
- **Integer types**: [signed and unsigned integers](./int-uint.md) (`UInt8`, `UInt16`, `UInt32`, `UInt64`, `UInt128`, `UInt256`, `Int8`, `Int16`, `Int32`, `Int64`, `Int128`, `Int256`)
|
||||
- **Floating-point numbers**: [floats](./float.md)(`Float32` and `Float64`) and [`Decimal` values](./decimal.md)
|
||||
- **Boolean**: ClickHouse has a [`Boolean` type](./boolean.md)
|
||||
- **Strings**: [`String`](./string.md) and [`FixedString`](./fixedstring.md)
|
||||
- **Dates**: use [`Date`](./date.md) and [`Date32`](./date32.md) for days, and [`DateTime`](./datetime.md) and [`DateTime64`](./datetime64.md) for instances in time
|
||||
- **Object**: the [`Object`](./json.md) stores a JSON document in a single column (deprecated)
|
||||
- **JSON**: the [`JSON` object](./newjson.md) stores a JSON document in a single column
|
||||
- **UUID**: a performant option for storing [`UUID` values](./uuid.md)
|
||||
- **Low cardinality types**: use an [`Enum`](./enum.md) when you have a handful of unique values, or use [`LowCardinality`](./lowcardinality.md) when you have up to 10,000 unique values of a column
|
||||
- **Arrays**: any column can be defined as an [`Array` of values](./array.md)
|
||||
- **Maps**: use [`Map`](./map.md) for storing key/value pairs
|
||||
- **Aggregation function types**: use [`SimpleAggregateFunction`](./simpleaggregatefunction.md) and [`AggregateFunction`](./aggregatefunction.md) for storing the intermediate status of aggregate function results
|
||||
- **Nested data structures**: A [`Nested` data structure](./nested-data-structures/index.md) is like a table inside a cell
|
||||
- **Tuples**: A [`Tuple` of elements](./tuple.md), each having an individual type.
|
||||
- **Nullable**: [`Nullable`](./nullable.md) allows you to store a value as `NULL` when a value is "missing" (instead of the column settings its default value for the data type)
|
||||
- **IP addresses**: use [`IPv4`](./ipv4.md) and [`IPv6`](./ipv6.md) to efficiently store IP addresses
|
||||
- **Geo types**: for [geographical data](./geo.md), including `Point`, `Ring`, `Polygon` and `MultiPolygon`
|
||||
- **Special data types**: including [`Expression`](./special-data-types/expression.md), [`Set`](./special-data-types/set.md), [`Nothing`](./special-data-types/nothing.md) and [`Interval`](./special-data-types/interval.md)
|
||||
System table [system.data_type_families](../../operations/system-tables/data_type_families.md#system_tables-data_type_families) provides an
|
||||
overview of all available data types.
|
||||
It also shows whether a data type is an alias to another data type and its name is case-sensitive (e.g. `bool` vs. `BOOL`).
|
||||
|
||||
@ -7,7 +7,7 @@ keywords: [object, data type]
|
||||
|
||||
# Object Data Type (deprecated)
|
||||
|
||||
**This feature is not production-ready and is now deprecated.** If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-formats/json/overview) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864).
|
||||
**This feature is not production-ready and deprecated.** If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-formats/json/overview) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864).
|
||||
|
||||
<hr />
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user