mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 17:41:59 +00:00
Merge branch 'master' into ascii_function
This commit is contained in:
李扬
GitHub
commit
206d46c605
2
.github/workflows/cancel.yml
vendored
2
.github/workflows/cancel.yml
vendored
@ -6,7 +6,7 @@ env:
|
||||
|
||||
on: # yamllint disable-line rule:truthy
|
||||
workflow_run:
|
||||
workflows: ["PullRequestCI", "ReleaseCI", "DocsCheck", "BackportPR"]
|
||||
workflows: ["PullRequestCI", "ReleaseBranchCI", "DocsCheck", "BackportPR"]
|
||||
types:
|
||||
- requested
|
||||
jobs:
|
||||
|
||||
55
.github/workflows/nightly.yml
vendored
55
.github/workflows/nightly.yml
vendored
@ -122,3 +122,58 @@ jobs:
|
||||
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
|
||||
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
|
||||
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
|
||||
SonarCloud:
|
||||
runs-on: [self-hosted, builder]
|
||||
env:
|
||||
SONAR_SCANNER_VERSION: 4.7.0.2747
|
||||
SONAR_SERVER_URL: "https://sonarcloud.io"
|
||||
BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
|
||||
CC: clang-15
|
||||
CXX: clang++-15
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
|
||||
submodules: true
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
with:
|
||||
java-version: 11
|
||||
- name: Download and set up sonar-scanner
|
||||
env:
|
||||
SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip
|
||||
run: |
|
||||
mkdir -p "$HOME/.sonar"
|
||||
curl -sSLo "$HOME/.sonar/sonar-scanner.zip" "${{ env.SONAR_SCANNER_DOWNLOAD_URL }}"
|
||||
unzip -o "$HOME/.sonar/sonar-scanner.zip" -d "$HOME/.sonar/"
|
||||
echo "$HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin" >> "$GITHUB_PATH"
|
||||
- name: Download and set up build-wrapper
|
||||
env:
|
||||
BUILD_WRAPPER_DOWNLOAD_URL: ${{ env.SONAR_SERVER_URL }}/static/cpp/build-wrapper-linux-x86.zip
|
||||
run: |
|
||||
curl -sSLo "$HOME/.sonar/build-wrapper-linux-x86.zip" "${{ env.BUILD_WRAPPER_DOWNLOAD_URL }}"
|
||||
unzip -o "$HOME/.sonar/build-wrapper-linux-x86.zip" -d "$HOME/.sonar/"
|
||||
echo "$HOME/.sonar/build-wrapper-linux-x86" >> "$GITHUB_PATH"
|
||||
- name: Set Up Build Tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -yq git cmake ccache python3 ninja-build
|
||||
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
|
||||
- name: Run build-wrapper
|
||||
run: |
|
||||
mkdir build
|
||||
cd build
|
||||
cmake ..
|
||||
cd ..
|
||||
build-wrapper-linux-x86-64 --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} cmake --build build/
|
||||
- name: Run sonar-scanner
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
run: |
|
||||
sonar-scanner \
|
||||
--define sonar.host.url="${{ env.SONAR_SERVER_URL }}" \
|
||||
--define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" \
|
||||
--define sonar.projectKey="ClickHouse_ClickHouse" \
|
||||
--define sonar.organization="clickhouse-java" \
|
||||
--define sonar.exclusions="**/*.java,**/*.ts,**/*.js,**/*.css,**/*.sql"
|
||||
|
||||
1
.github/workflows/pull_request.yml
vendored
1
.github/workflows/pull_request.yml
vendored
@ -2023,6 +2023,7 @@ jobs:
|
||||
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
TestsBugfixCheck:
|
||||
needs: [CheckLabels, StyleCheck]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
- name: Set envs
|
||||
|
||||
4
.snyk
Normal file
4
.snyk
Normal file
@ -0,0 +1,4 @@
|
||||
# Snyk (https://snyk.io) policy file
|
||||
exclude:
|
||||
global:
|
||||
- tests/**
|
||||
@ -5,7 +5,7 @@ ClickHouse® is an open-source column-oriented database management system that a
|
||||
## Useful Links
|
||||
|
||||
* [Official website](https://clickhouse.com/) has a quick high-level overview of ClickHouse on the main page.
|
||||
* [ClickHouse Cloud](https://clickhouse.com/cloud) ClickHouse as a service, built by the creators and maintainers.
|
||||
* [ClickHouse Cloud](https://clickhouse.cloud) ClickHouse as a service, built by the creators and maintainers.
|
||||
* [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster.
|
||||
* [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information.
|
||||
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
|
||||
@ -16,5 +16,6 @@ ClickHouse® is an open-source column-oriented database management system that a
|
||||
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
|
||||
|
||||
## Upcoming events
|
||||
* [**v22.10 Release Webinar**](https://clickhouse.com/company/events/v22-10-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
|
||||
* [**Introducing ClickHouse Cloud**](https://clickhouse.com/company/events/cloud-beta) Introducing ClickHouse as a service, built by creators and maintainers of the fastest OLAP database on earth. Join Tanya Bragin for a detailed walkthrough of ClickHouse Cloud capabilities, as well as a peek behind the curtain to understand the unique architecture that makes our service tick.
|
||||
* [**v22.11 Release Webinar**](https://clickhouse.com/company/events/v22-11-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
|
||||
* [**ClickHouse Meetup at the Deutsche Bank office in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/289311596/) Hear from Deutsche Bank on why they chose ClickHouse for big sensitive data in a regulated environment. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management.
|
||||
* [**AWS re:Invent**](https://clickhouse.com/company/events/aws-reinvent) Core members of the ClickHouse team -- including 2 of our founders -- will be at re:Invent from November 29 to December 3. We are available on the show floor, but are also determining interest in holding an event during the time there.
|
||||
|
||||
2
contrib/cctz
vendored
2
contrib/cctz
vendored
@ -1 +1 @@
|
||||
Subproject commit 7a454c25c7d16053bcd327cdd16329212a08fa4a
|
||||
Subproject commit 5c8528fb35e89ee0b3a7157490423fba0d4dd7b5
|
||||
@ -21,6 +21,9 @@ set (LLVM_INCLUDE_DIRS
|
||||
"${ClickHouse_BINARY_DIR}/contrib/llvm-project/llvm/include"
|
||||
)
|
||||
set (LLVM_LIBRARY_DIRS "${ClickHouse_BINARY_DIR}/contrib/llvm-project/llvm")
|
||||
# NOTE: You should not remove this line since otherwise it will use default 20,
|
||||
# and llvm cannot be compiled with bundled libcxx and 20 standard.
|
||||
set (CMAKE_CXX_STANDARD 14)
|
||||
|
||||
# This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles.
|
||||
set (REQUIRED_LLVM_LIBRARIES
|
||||
|
||||
@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="22.10.1.1877"
|
||||
ARG VERSION="22.10.2.11"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
# user/group precreated explicitly with fixed uid/gid on purpose.
|
||||
|
||||
@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
|
||||
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
|
||||
ARG VERSION="22.10.1.1877"
|
||||
ARG VERSION="22.10.2.11"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
# set non-empty deb_location_url url to create a docker image
|
||||
|
||||
@ -212,4 +212,4 @@ Templates:
|
||||
|
||||
## How to Build Documentation
|
||||
|
||||
You can build your documentation manually by following the instructions in [docs/tools/README.md](../docs/tools/README.md). Also, our CI runs the documentation build after the `documentation` label is added to PR. You can see the results of a build in the GitHub interface. If you have no permissions to add labels, a reviewer of your PR will add it.
|
||||
You can build your documentation manually by following the instructions in the docs repo [contrib-writing-guide](https://github.com/ClickHouse/clickhouse-docs/blob/main/contrib-writing-guide.md). Also, our CI runs the documentation build after the `documentation` label is added to PR. You can see the results of a build in the GitHub interface. If you have no permissions to add labels, a reviewer of your PR will add it.
|
||||
|
||||
18
docs/changelogs/v22.10.2.11-stable.md
Normal file
18
docs/changelogs/v22.10.2.11-stable.md
Normal file
@ -0,0 +1,18 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2022
|
||||
---
|
||||
|
||||
# 2022 Changelog
|
||||
|
||||
### ClickHouse release v22.10.2.11-stable (d2bfcaba002) FIXME as compared to v22.10.1.1877-stable (98ab5a3c189)
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
|
||||
|
||||
* Backported in [#42750](https://github.com/ClickHouse/ClickHouse/issues/42750): A segmentation fault related to DNS & c-ares has been reported. The below error ocurred in multiple threads: ``` 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008088 [ 356 ] {} <Fatal> BaseDaemon: ######################################## 2022-09-28 15:41:19.008,"2022.09.28 15:41:19.008147 [ 356 ] {} <Fatal> BaseDaemon: (version 22.8.5.29 (official build), build id: 92504ACA0B8E2267) (from thread 353) (no query) Received signal Segmentation fault (11)" 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008196 [ 356 ] {} <Fatal> BaseDaemon: Address: 0xf Access: write. Address not mapped to object. 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008216 [ 356 ] {} <Fatal> BaseDaemon: Stack trace: 0x188f8212 0x1626851b 0x1626a69e 0x16269b3f 0x16267eab 0x13cf8284 0x13d24afc 0x13c5217e 0x14ec2495 0x15ba440f 0x15b9d13b 0x15bb2699 0x1891ccb3 0x1891e00d 0x18ae0769 0x18ade022 0x7f76aa985609 0x7f76aa8aa133 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008274 [ 356 ] {} <Fatal> BaseDaemon: 2. Poco::Net::IPAddress::family() const @ 0x188f8212 in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008297 [ 356 ] {} <Fatal> BaseDaemon: 3. ? @ 0x1626851b in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008309 [ 356 ] {} <Fatal> BaseDaemon: 4. ? @ 0x1626a69e in /usr/bin/clickhouse ```. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* Backported in [#42793](https://github.com/ClickHouse/ClickHouse/issues/42793): Fix a bug in ParserFunction that could have led to a segmentation fault. [#42724](https://github.com/ClickHouse/ClickHouse/pull/42724) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Always run `BuilderReport` and `BuilderSpecialReport` in all CI types [#42684](https://github.com/ClickHouse/ClickHouse/pull/42684) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
26
docs/changelogs/v22.3.14.18-lts.md
Normal file
26
docs/changelogs/v22.3.14.18-lts.md
Normal file
@ -0,0 +1,26 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2022
|
||||
---
|
||||
|
||||
# 2022 Changelog
|
||||
|
||||
### ClickHouse release v22.3.14.18-lts (642946f61b2) FIXME as compared to v22.3.13.80-lts (e2708b01fba)
|
||||
|
||||
#### Bug Fix
|
||||
* Backported in [#42432](https://github.com/ClickHouse/ClickHouse/issues/42432): - Choose correct aggregation method for LowCardinality with BigInt. [#42342](https://github.com/ClickHouse/ClickHouse/pull/42342) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#42328](https://github.com/ClickHouse/ClickHouse/issues/42328): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)).
|
||||
* Backported in [#42358](https://github.com/ClickHouse/ClickHouse/issues/42358): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
|
||||
|
||||
* Backported in [#42298](https://github.com/ClickHouse/ClickHouse/issues/42298): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#42592](https://github.com/ClickHouse/ClickHouse/issues/42592): This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Add a warning message to release.py script, require release type [#41975](https://github.com/ClickHouse/ClickHouse/pull/41975) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Revert [#27787](https://github.com/ClickHouse/ClickHouse/issues/27787) [#42136](https://github.com/ClickHouse/ClickHouse/pull/42136) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
29
docs/changelogs/v22.3.14.23-lts.md
Normal file
29
docs/changelogs/v22.3.14.23-lts.md
Normal file
@ -0,0 +1,29 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2022
|
||||
---
|
||||
|
||||
# 2022 Changelog
|
||||
|
||||
### ClickHouse release v22.3.14.23-lts (74956bfee4d) FIXME as compared to v22.3.13.80-lts (e2708b01fba)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#42527](https://github.com/ClickHouse/ClickHouse/issues/42527): Fix issue with passing MySQL timeouts for MySQL database engine and MySQL table function. Closes [#34168](https://github.com/ClickHouse/ClickHouse/issues/34168)?notification_referrer_id=NT_kwDOAzsV57MzMDMxNjAzNTY5OjU0MjAzODc5. [#40751](https://github.com/ClickHouse/ClickHouse/pull/40751) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
|
||||
#### Bug Fix
|
||||
* Backported in [#42432](https://github.com/ClickHouse/ClickHouse/issues/42432): - Choose correct aggregation method for LowCardinality with BigInt. [#42342](https://github.com/ClickHouse/ClickHouse/pull/42342) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#42328](https://github.com/ClickHouse/ClickHouse/issues/42328): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)).
|
||||
* Backported in [#42358](https://github.com/ClickHouse/ClickHouse/issues/42358): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
|
||||
|
||||
* Backported in [#42298](https://github.com/ClickHouse/ClickHouse/issues/42298): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#42592](https://github.com/ClickHouse/ClickHouse/issues/42592): This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Add a warning message to release.py script, require release type [#41975](https://github.com/ClickHouse/ClickHouse/pull/41975) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Revert [#27787](https://github.com/ClickHouse/ClickHouse/issues/27787) [#42136](https://github.com/ClickHouse/ClickHouse/pull/42136) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
@ -105,7 +105,7 @@ ninja
|
||||
Example for Fedora Rawhide:
|
||||
``` bash
|
||||
sudo yum update
|
||||
yum --nogpg install git cmake make clang-c++ python3
|
||||
sudo yum --nogpg install git cmake make clang python3 ccache
|
||||
git clone --recursive https://github.com/ClickHouse/ClickHouse.git
|
||||
mkdir build && cd build
|
||||
cmake ../ClickHouse
|
||||
|
||||
@ -139,7 +139,7 @@ The following settings can be specified in configuration file for given endpoint
|
||||
- `use_environment_credentials` — If set to `true`, S3 client will try to obtain credentials from environment variables and [Amazon EC2](https://en.wikipedia.org/wiki/Amazon_Elastic_Compute_Cloud) metadata for given endpoint. Optional, default value is `false`.
|
||||
- `region` — Specifies S3 region name. Optional.
|
||||
- `use_insecure_imds_request` — If set to `true`, S3 client will use insecure IMDS request while obtaining credentials from Amazon EC2 metadata. Optional, default value is `false`.
|
||||
- `header` — Adds specified HTTP header to a request to given endpoint. Optional, can be speficied multiple times.
|
||||
- `header` — Adds specified HTTP header to a request to given endpoint. Optional, can be specified multiple times.
|
||||
- `server_side_encryption_customer_key_base64` — If specified, required headers for accessing S3 objects with SSE-C encryption will be set. Optional.
|
||||
- `max_single_read_retries` — The maximum number of attempts during single read. Default value is `4`. Optional.
|
||||
|
||||
|
||||
@ -33,7 +33,7 @@ CREATE TABLE trips (
|
||||
tip_amount Float32,
|
||||
tolls_amount Float32,
|
||||
total_amount Float32,
|
||||
payment_type Enum('CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4),
|
||||
payment_type Enum('CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4, 'UNK' = 5),
|
||||
pickup_ntaname LowCardinality(String),
|
||||
dropoff_ntaname LowCardinality(String)
|
||||
)
|
||||
@ -63,7 +63,7 @@ SELECT
|
||||
payment_type,
|
||||
pickup_ntaname,
|
||||
dropoff_ntaname
|
||||
FROM url(
|
||||
FROM s3(
|
||||
'https://datasets-documentation.s3.eu-west-3.amazonaws.com/nyc-taxi/trips_{0..2}.gz',
|
||||
'TabSeparatedWithNames'
|
||||
)
|
||||
|
||||
@ -128,6 +128,24 @@ clickhouse-client # or "clickhouse-client --password" if you set up a password.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Migration Method for installing the deb-packages</summary>
|
||||
|
||||
```bash
|
||||
sudo apt-key del E0C56BD4
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
|
||||
echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
|
||||
/etc/apt/sources.list.d/clickhouse.list
|
||||
sudo apt-get update
|
||||
|
||||
sudo apt-get install -y clickhouse-server clickhouse-client
|
||||
|
||||
sudo service clickhouse-server start
|
||||
clickhouse-client # or "clickhouse-client --password" if you set up a password.
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
You can replace `stable` with `lts` to use different [release kinds](/docs/en/faq/operations/production.md) based on your needs.
|
||||
|
||||
You can also download and install packages manually from [here](https://packages.clickhouse.com/deb/pool/main/c/).
|
||||
|
||||
@ -1,10 +1,7 @@
|
||||
---
|
||||
slug: /en/operations/update
|
||||
sidebar_position: 47
|
||||
sidebar_label: ClickHouse Upgrade
|
||||
---
|
||||
|
||||
# ClickHouse Upgrade
|
||||
[//]: # (This file is included in Manage > Updates)
|
||||
|
||||
## Self-managed ClickHouse Upgrade
|
||||
|
||||
If ClickHouse was installed from `deb` packages, execute the following commands on the server:
|
||||
|
||||
@ -309,7 +309,7 @@ Sessions with Ephemerals (1):
|
||||
/clickhouse/task_queue/ddl
|
||||
```
|
||||
|
||||
## [experimental] Migration from ZooKeeper {#migration-from-zookeeper}
|
||||
## Migration from ZooKeeper {#migration-from-zookeeper}
|
||||
|
||||
Seamlessly migration from ZooKeeper to ClickHouse Keeper is impossible you have to stop your ZooKeeper cluster, convert data and start ClickHouse Keeper. `clickhouse-keeper-converter` tool allows converting ZooKeeper logs and snapshots to ClickHouse Keeper snapshot. It works only with ZooKeeper > 3.4. Steps for migration:
|
||||
|
||||
|
||||
@ -178,7 +178,7 @@ Columns:
|
||||
- `view_definition` ([String](../../sql-reference/data-types/string.md)) — `SELECT` query for view.
|
||||
- `check_option` ([String](../../sql-reference/data-types/string.md)) — `NONE`, no checking.
|
||||
- `is_updatable` ([Enum8](../../sql-reference/data-types/enum.md)) — `NO`, the view is not updated.
|
||||
- `is_insertable_into` ([Enum8](../../sql-reference/data-types/enum.md)) — Shows whether the created view is [materialized](../../sql-reference/statements/create/view/#materialized). Possible values:
|
||||
- `is_insertable_into` ([Enum8](../../sql-reference/data-types/enum.md)) — Shows whether the created view is [materialized](../../sql-reference/statements/create/view.md/#materialized-view). Possible values:
|
||||
- `NO` — The created view is not materialized.
|
||||
- `YES` — The created view is materialized.
|
||||
- `is_trigger_updatable` ([Enum8](../../sql-reference/data-types/enum.md)) — `NO`, the trigger is not updated.
|
||||
|
||||
@ -68,6 +68,5 @@ thread_id: 54
|
||||
|
||||
**See Also**
|
||||
|
||||
- [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system/#query-language-system-replicated)
|
||||
- [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system.md/#managing-replicatedmergetree-tables)
|
||||
|
||||
[Original article](https://clickhouse.com/docs/en/operations/system_tables/replicated_fetches) <!--hide-->
|
||||
|
||||
@ -303,17 +303,25 @@ or
|
||||
CREATE DICTIONARY somedict (
|
||||
id UInt64,
|
||||
first Date,
|
||||
last Date
|
||||
last Date,
|
||||
advertiser_id UInt64
|
||||
)
|
||||
PRIMARY KEY id
|
||||
SOURCE(CLICKHOUSE(TABLE 'date_table'))
|
||||
LIFETIME(MIN 1 MAX 1000)
|
||||
LAYOUT(RANGE_HASHED())
|
||||
RANGE(MIN first MAX last)
|
||||
```
|
||||
|
||||
To work with these dictionaries, you need to pass an additional argument to the `dictGetT` function, for which a range is selected:
|
||||
To work with these dictionaries, you need to pass an additional argument to the `dictGet` function, for which a range is selected:
|
||||
|
||||
``` sql
|
||||
dictGetT('dict_name', 'attr_name', id, date)
|
||||
dictGet('dict_name', 'attr_name', id, date)
|
||||
```
|
||||
Query example:
|
||||
|
||||
``` sql
|
||||
SELECT dictGet('somedict', 'advertiser_id', 1, '2022-10-20 23:20:10.000'::DateTime64::UInt64);
|
||||
```
|
||||
|
||||
This function returns the value for the specified `id`s and the date range that includes the passed date.
|
||||
|
||||
@ -14,8 +14,10 @@ Example of a polygon dictionary configuration:
|
||||
<dictionary>
|
||||
<structure>
|
||||
<key>
|
||||
<name>key</name>
|
||||
<type>Array(Array(Array(Array(Float64))))</type>
|
||||
<attribute>
|
||||
<name>key</name>
|
||||
<type>Array(Array(Array(Array(Float64))))</type>
|
||||
</attribute>
|
||||
</key>
|
||||
|
||||
<attribute>
|
||||
|
||||
@ -1068,7 +1068,7 @@ Example:
|
||||
SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600));
|
||||
SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299);
|
||||
SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0));
|
||||
```
|
||||
```
|
||||
``` text
|
||||
┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐
|
||||
│ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │
|
||||
@ -1244,7 +1244,7 @@ Result:
|
||||
└──────────────────────────┘
|
||||
```
|
||||
|
||||
When there are two arguments: first is an [Integer](../../sql-reference/data-types/int-uint.md) or [DateTime](../../sql-reference/data-types/datetime.md), second is a constant format string — it acts in the same way as [formatDateTime](#formatdatetime) and return [String](../../sql-reference/data-types/string.md#string) type.
|
||||
When there are two or three arguments, the first an [Integer](../../sql-reference/data-types/int-uint.md), [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md), the second a constant format string and the third an optional constant time zone string — it acts in the same way as [formatDateTime](#formatdatetime) and return [String](../../sql-reference/data-types/string.md#string) type.
|
||||
|
||||
For example:
|
||||
|
||||
|
||||
@ -107,7 +107,7 @@ ALTER TABLE visits RENAME COLUMN webBrowser TO browser
|
||||
CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name
|
||||
```
|
||||
|
||||
Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to set the partition expression](partition.md#how-to-set-partition-expression).
|
||||
|
||||
If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist.
|
||||
|
||||
@ -204,8 +204,9 @@ It is used if it is necessary to add or update a column with a complicated expre
|
||||
Syntax:
|
||||
|
||||
```sql
|
||||
ALTER TABLE table MATERIALIZE COLUMN col;
|
||||
ALTER TABLE [db.]table [ON CLUSTER cluster] MATERIALIZE COLUMN col [IN PARTITION partition | IN PARTITION ID 'partition_id'];
|
||||
```
|
||||
- If you specify a PARTITION, a column will be materialized with only the specified partition.
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
@ -39,7 +39,7 @@ ALTER TABLE mt DETACH PARTITION '2020-11-21';
|
||||
ALTER TABLE mt DETACH PART 'all_2_2_0';
|
||||
```
|
||||
|
||||
Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
Read about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
|
||||
|
||||
After the query is executed, you can do whatever you want with the data in the `detached` directory — delete it from the file system, or just leave it.
|
||||
|
||||
@ -53,7 +53,7 @@ ALTER TABLE table_name [ON CLUSTER cluster] DROP PARTITION|PART partition_expr
|
||||
|
||||
Deletes the specified partition from the table. This query tags the partition as inactive and deletes data completely, approximately in 10 minutes.
|
||||
|
||||
Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
Read about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
|
||||
|
||||
The query is replicated – it deletes data on all replicas.
|
||||
|
||||
@ -71,7 +71,7 @@ ALTER TABLE table_name [ON CLUSTER cluster] DROP DETACHED PARTITION|PART partiti
|
||||
```
|
||||
|
||||
Removes the specified part or all parts of the specified partition from `detached`.
|
||||
Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
Read more about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
|
||||
|
||||
## ATTACH PARTITION\|PART
|
||||
|
||||
@ -86,7 +86,7 @@ ALTER TABLE visits ATTACH PARTITION 201901;
|
||||
ALTER TABLE visits ATTACH PART 201901_2_2_0;
|
||||
```
|
||||
|
||||
Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
Read more about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
|
||||
|
||||
This query is replicated. The replica-initiator checks whether there is data in the `detached` directory.
|
||||
If data exists, the query checks its integrity. If everything is correct, the query adds the data to the table.
|
||||
@ -166,7 +166,7 @@ This query creates a local backup of a specified partition. If the `PARTITION` c
|
||||
The entire backup process is performed without stopping the server.
|
||||
:::
|
||||
|
||||
Note that for old-styled tables you can specify the prefix of the partition name (for example, `2019`) - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
Note that for old-styled tables you can specify the prefix of the partition name (for example, `2019`) - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
|
||||
|
||||
At the time of execution, for a data snapshot, the query creates hardlinks to a table data. Hardlinks are placed in the directory `/var/lib/clickhouse/shadow/N/...`, where:
|
||||
|
||||
|
||||
@ -7,18 +7,26 @@ title: "Manipulating Projections"
|
||||
|
||||
The following operations with [projections](../../../engines/table-engines/mergetree-family/mergetree.md#projections) are available:
|
||||
|
||||
- `ALTER TABLE [db].name ADD PROJECTION name ( SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY] )` - Adds projection description to tables metadata.
|
||||
## ADD PROJECTION
|
||||
|
||||
- `ALTER TABLE [db].name DROP PROJECTION name` - Removes projection description from tables metadata and deletes projection files from disk. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
`ALTER TABLE [db].name ADD PROJECTION name ( SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY] )` - Adds projection description to tables metadata.
|
||||
|
||||
- `ALTER TABLE [db.]table MATERIALIZE PROJECTION name IN PARTITION partition_name` - The query rebuilds the projection `name` in the partition `partition_name`. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
## DROP PROJECTION
|
||||
|
||||
- `ALTER TABLE [db.]table CLEAR PROJECTION name IN PARTITION partition_name` - Deletes projection files from disk without removing description. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
`ALTER TABLE [db].name DROP PROJECTION name` - Removes projection description from tables metadata and deletes projection files from disk. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
## MATERIALIZE PROJECTION
|
||||
|
||||
`ALTER TABLE [db.]table MATERIALIZE PROJECTION name IN PARTITION partition_name` - The query rebuilds the projection `name` in the partition `partition_name`. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
## CLEAR PROJECTION
|
||||
|
||||
`ALTER TABLE [db.]table CLEAR PROJECTION name IN PARTITION partition_name` - Deletes projection files from disk without removing description. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
|
||||
The commands `ADD`, `DROP` and `CLEAR` are lightweight in a sense that they only change metadata or remove files.
|
||||
|
||||
Also, they are replicated, syncing projections metadata via ZooKeeper.
|
||||
Also, they are replicated, syncing projections metadata via ClickHouse Keeper or ZooKeeper.
|
||||
|
||||
:::note
|
||||
Projection manipulation is supported only for tables with [`*MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) engine (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) variants).
|
||||
|
||||
@ -31,7 +31,7 @@ By default, ClickHouse uses its own [Atomic](../../../engines/database-engines/a
|
||||
|
||||
### COMMENT
|
||||
|
||||
You can add a comment to the database when you creating it.
|
||||
You can add a comment to the database when you are creating it.
|
||||
|
||||
The comment is supported for all database engines.
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@ sidebar_position: 38
|
||||
sidebar_label: FUNCTION
|
||||
---
|
||||
|
||||
# CREATE FUNCTION
|
||||
# CREATE FUNCTION — user defined function (UDF)
|
||||
|
||||
Creates a user defined function from a lambda expression. The expression must consist of function parameters, constants, operators, or other function calls.
|
||||
|
||||
|
||||
@ -22,7 +22,7 @@ The `OPTIMIZE` query is supported for [MergeTree](../../engines/table-engines/me
|
||||
When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all replicas (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `2`) or on current replica (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `1`).
|
||||
|
||||
- If `OPTIMIZE` does not perform a merge for any reason, it does not notify the client. To enable notifications, use the [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting.
|
||||
- If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](../../sql-reference/statements/alter/index.md#alter-how-to-specify-part-expr).
|
||||
- If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](alter/partition.md#how-to-set-partition-expression).
|
||||
- If you specify `FINAL`, optimization is performed even when all the data is already in one part. Also merge is forced even if concurrent merges are performed.
|
||||
- If you specify `DEDUPLICATE`, then completely identical rows (unless by-clause is specified) will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine.
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: INTERSECT
|
||||
|
||||
The `INTERSECT` clause returns only those rows that result from both the first and the second queries. The queries must match the number of columns, order, and type. The result of `INTERSECT` can contain duplicate rows.
|
||||
|
||||
Multiple `INTERSECT` statements are executes left to right if parenthesis are not specified. The `INTERSECT` operator has a higher priority than the `UNION` and `EXCEPT` clause.
|
||||
Multiple `INTERSECT` statements are executed left to right if parentheses are not specified. The `INTERSECT` operator has a higher priority than the `UNION` and `EXCEPT` clauses.
|
||||
|
||||
|
||||
``` sql
|
||||
|
||||
@ -1126,8 +1126,7 @@ SELECT FROM_UNIXTIME(423543535);
|
||||
└──────────────────────────┘
|
||||
```
|
||||
|
||||
В случае, когда есть два аргумента: первый типа [Integer](../../sql-reference/data-types/int-uint.md) или [DateTime](../../sql-reference/data-types/datetime.md), а второй является строкой постоянного формата — функция работает также, как [formatDateTime](#formatdatetime), и возвращает значение типа [String](../../sql-reference/data-types/string.md#string).
|
||||
|
||||
В случае, когда есть два или три аргумента: первый типа [Integer](../../sql-reference/data-types/int-uint.md), [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) или [DateTime64](../../sql-reference/data-types/datetime64.md), а второй является строкой постоянного формата и третий является строкой постоянной временной зоны — функция работает также, как [formatDateTime](#formatdatetime), и возвращает значение типа [String](../../sql-reference/data-types/string.md#string).
|
||||
|
||||
Запрос:
|
||||
|
||||
|
||||
@ -1621,34 +1621,7 @@ void QueryAnalyzer::validateTableExpressionModifiers(const QueryTreeNodePtr & ta
|
||||
table_expression_node->formatASTForErrorMessage(),
|
||||
scope.scope_node->formatASTForErrorMessage());
|
||||
|
||||
if (query_node || union_node)
|
||||
{
|
||||
auto table_expression_modifiers = query_node ? query_node->getTableExpressionModifiers() : union_node->getTableExpressionModifiers();
|
||||
|
||||
if (table_expression_modifiers.has_value())
|
||||
{
|
||||
String table_expression_modifiers_error_message;
|
||||
|
||||
if (table_expression_modifiers->hasFinal())
|
||||
{
|
||||
table_expression_modifiers_error_message += "FINAL";
|
||||
|
||||
if (table_expression_modifiers->hasSampleSizeRatio())
|
||||
table_expression_modifiers_error_message += ", SAMPLE";
|
||||
}
|
||||
else if (table_expression_modifiers->hasSampleSizeRatio())
|
||||
{
|
||||
table_expression_modifiers_error_message += "SAMPLE";
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
|
||||
"Table expression modifiers {} are not supported for subquery {}. In scope {}",
|
||||
table_expression_modifiers_error_message,
|
||||
table_expression_node->formatASTForErrorMessage(),
|
||||
scope.scope_node->formatASTForErrorMessage());
|
||||
}
|
||||
}
|
||||
else if (table_node || table_function_node)
|
||||
if (table_node || table_function_node)
|
||||
{
|
||||
auto table_expression_modifiers = table_node ? table_node->getTableExpressionModifiers() : table_function_node->getTableExpressionModifiers();
|
||||
|
||||
@ -4661,17 +4634,23 @@ void QueryAnalyzer::initializeQueryJoinTreeNode(QueryTreeNodePtr & join_tree_nod
|
||||
|
||||
auto table_expression_modifiers = from_table_identifier.getTableExpressionModifiers();
|
||||
|
||||
if (auto * resolved_identifier_query_node = resolved_identifier->as<QueryNode>())
|
||||
auto * resolved_identifier_query_node = resolved_identifier->as<QueryNode>();
|
||||
auto * resolved_identifier_union_node = resolved_identifier->as<UnionNode>();
|
||||
|
||||
if (resolved_identifier_query_node || resolved_identifier_union_node)
|
||||
{
|
||||
resolved_identifier_query_node->setIsCTE(false);
|
||||
if (resolved_identifier_query_node)
|
||||
resolved_identifier_query_node->setIsCTE(false);
|
||||
else
|
||||
resolved_identifier_union_node->setIsCTE(false);
|
||||
|
||||
if (table_expression_modifiers.has_value())
|
||||
resolved_identifier_query_node->setTableExpressionModifiers(*table_expression_modifiers);
|
||||
}
|
||||
else if (auto * resolved_identifier_union_node = resolved_identifier->as<UnionNode>())
|
||||
{
|
||||
resolved_identifier_union_node->setIsCTE(false);
|
||||
if (table_expression_modifiers.has_value())
|
||||
resolved_identifier_union_node->setTableExpressionModifiers(*table_expression_modifiers);
|
||||
{
|
||||
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
|
||||
"Table expression modifiers {} are not supported for subquery {}",
|
||||
table_expression_modifiers->formatForErrorMessage(),
|
||||
resolved_identifier->formatASTForErrorMessage());
|
||||
}
|
||||
}
|
||||
else if (auto * resolved_identifier_table_node = resolved_identifier->as<TableNode>())
|
||||
{
|
||||
|
||||
@ -74,12 +74,6 @@ void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s
|
||||
buffer << ", constant_value_type: " << constant_value->getType()->getName();
|
||||
}
|
||||
|
||||
if (table_expression_modifiers)
|
||||
{
|
||||
buffer << ", ";
|
||||
table_expression_modifiers->dump(buffer);
|
||||
}
|
||||
|
||||
if (hasWith())
|
||||
{
|
||||
buffer << '\n' << std::string(indent + 2, ' ') << "WITH\n";
|
||||
@ -195,13 +189,6 @@ bool QueryNode::isEqualImpl(const IQueryTreeNode & rhs) const
|
||||
else if (!constant_value && rhs_typed.constant_value)
|
||||
return false;
|
||||
|
||||
if (table_expression_modifiers && rhs_typed.table_expression_modifiers && table_expression_modifiers != rhs_typed.table_expression_modifiers)
|
||||
return false;
|
||||
else if (table_expression_modifiers && !rhs_typed.table_expression_modifiers)
|
||||
return false;
|
||||
else if (!table_expression_modifiers && rhs_typed.table_expression_modifiers)
|
||||
return false;
|
||||
|
||||
return is_subquery == rhs_typed.is_subquery &&
|
||||
is_cte == rhs_typed.is_cte &&
|
||||
cte_name == rhs_typed.cte_name &&
|
||||
@ -250,9 +237,6 @@ void QueryNode::updateTreeHashImpl(HashState & state) const
|
||||
state.update(constant_value_type_name.size());
|
||||
state.update(constant_value_type_name);
|
||||
}
|
||||
|
||||
if (table_expression_modifiers)
|
||||
table_expression_modifiers->updateTreeHash(state);
|
||||
}
|
||||
|
||||
QueryTreeNodePtr QueryNode::cloneImpl() const
|
||||
@ -270,7 +254,6 @@ QueryTreeNodePtr QueryNode::cloneImpl() const
|
||||
result_query_node->cte_name = cte_name;
|
||||
result_query_node->projection_columns = projection_columns;
|
||||
result_query_node->constant_value = constant_value;
|
||||
result_query_node->table_expression_modifiers = table_expression_modifiers;
|
||||
|
||||
return result_query_node;
|
||||
}
|
||||
|
||||
@ -176,24 +176,6 @@ public:
|
||||
is_group_by_with_grouping_sets = is_group_by_with_grouping_sets_value;
|
||||
}
|
||||
|
||||
/// Return true if query node has table expression modifiers, false otherwise
|
||||
bool hasTableExpressionModifiers() const
|
||||
{
|
||||
return table_expression_modifiers.has_value();
|
||||
}
|
||||
|
||||
/// Get table expression modifiers
|
||||
const std::optional<TableExpressionModifiers> & getTableExpressionModifiers() const
|
||||
{
|
||||
return table_expression_modifiers;
|
||||
}
|
||||
|
||||
/// Set table expression modifiers
|
||||
void setTableExpressionModifiers(TableExpressionModifiers table_expression_modifiers_value)
|
||||
{
|
||||
table_expression_modifiers = std::move(table_expression_modifiers_value);
|
||||
}
|
||||
|
||||
/// Returns true if query node WITH section is not empty, false otherwise
|
||||
bool hasWith() const
|
||||
{
|
||||
@ -602,7 +584,6 @@ private:
|
||||
std::string cte_name;
|
||||
NamesAndTypes projection_columns;
|
||||
ConstantValuePtr constant_value;
|
||||
std::optional<TableExpressionModifiers> table_expression_modifiers;
|
||||
SettingsChanges settings_changes;
|
||||
|
||||
static constexpr size_t with_child_index = 0;
|
||||
|
||||
@ -145,12 +145,10 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectWithUnionExpression(const ASTPtr &
|
||||
if (select_lists.children.size() == 1)
|
||||
return buildSelectOrUnionExpression(select_lists.children[0], is_subquery, cte_name);
|
||||
|
||||
auto union_node = std::make_shared<UnionNode>();
|
||||
auto union_node = std::make_shared<UnionNode>(select_with_union_query_typed.union_mode);
|
||||
union_node->setIsSubquery(is_subquery);
|
||||
union_node->setIsCTE(!cte_name.empty());
|
||||
union_node->setCTEName(cte_name);
|
||||
union_node->setUnionMode(select_with_union_query_typed.union_mode);
|
||||
union_node->setUnionModes(select_with_union_query_typed.list_of_modes);
|
||||
union_node->setOriginalAST(select_with_union_query);
|
||||
|
||||
size_t select_lists_children_size = select_lists.children.size();
|
||||
@ -173,23 +171,22 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr
|
||||
if (select_lists.size() == 1)
|
||||
return buildSelectExpression(select_lists[0], is_subquery, cte_name);
|
||||
|
||||
auto union_node = std::make_shared<UnionNode>();
|
||||
union_node->setIsSubquery(is_subquery);
|
||||
union_node->setIsCTE(!cte_name.empty());
|
||||
union_node->setCTEName(cte_name);
|
||||
|
||||
SelectUnionMode union_mode;
|
||||
if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::INTERSECT_ALL)
|
||||
union_node->setUnionMode(SelectUnionMode::INTERSECT_ALL);
|
||||
union_mode = SelectUnionMode::INTERSECT_ALL;
|
||||
else if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::INTERSECT_DISTINCT)
|
||||
union_node->setUnionMode(SelectUnionMode::INTERSECT_DISTINCT);
|
||||
union_mode = SelectUnionMode::INTERSECT_DISTINCT;
|
||||
else if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::EXCEPT_ALL)
|
||||
union_node->setUnionMode(SelectUnionMode::EXCEPT_ALL);
|
||||
union_mode = SelectUnionMode::EXCEPT_ALL;
|
||||
else if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::EXCEPT_DISTINCT)
|
||||
union_node->setUnionMode(SelectUnionMode::EXCEPT_DISTINCT);
|
||||
union_mode = SelectUnionMode::EXCEPT_DISTINCT;
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "UNION type is not initialized");
|
||||
|
||||
union_node->setUnionModes(SelectUnionModes(select_lists.size() - 1, union_node->getUnionMode()));
|
||||
auto union_node = std::make_shared<UnionNode>(union_mode);
|
||||
union_node->setIsSubquery(is_subquery);
|
||||
union_node->setIsCTE(!cte_name.empty());
|
||||
union_node->setCTEName(cte_name);
|
||||
union_node->setOriginalAST(select_intersect_except_query);
|
||||
|
||||
size_t select_lists_size = select_lists.size();
|
||||
@ -676,14 +673,10 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
|
||||
|
||||
if (table_expression_modifiers)
|
||||
{
|
||||
if (auto * query_node = node->as<QueryNode>())
|
||||
query_node->setTableExpressionModifiers(*table_expression_modifiers);
|
||||
else if (auto * union_node = node->as<UnionNode>())
|
||||
union_node->setTableExpressionModifiers(*table_expression_modifiers);
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Unexpected table expression subquery node. Expected union or query. Actual {}",
|
||||
node->formatASTForErrorMessage());
|
||||
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
|
||||
"Table expression modifiers {} are not supported for subquery {}",
|
||||
table_expression_modifiers->formatForErrorMessage(),
|
||||
node->formatASTForErrorMessage());
|
||||
}
|
||||
|
||||
table_expressions.push_back(std::move(node));
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -39,4 +40,27 @@ void TableExpressionModifiers::updateTreeHash(SipHash & hash_state) const
|
||||
}
|
||||
}
|
||||
|
||||
String TableExpressionModifiers::formatForErrorMessage() const
|
||||
{
|
||||
WriteBufferFromOwnString buffer;
|
||||
if (has_final)
|
||||
buffer << "FINAL";
|
||||
|
||||
if (sample_size_ratio)
|
||||
{
|
||||
if (has_final)
|
||||
buffer << ' ';
|
||||
buffer << "SAMPLE " << ASTSampleRatio::toString(*sample_size_ratio);
|
||||
}
|
||||
|
||||
if (sample_offset_ratio)
|
||||
{
|
||||
if (has_final || sample_size_ratio)
|
||||
buffer << ' ';
|
||||
buffer << "OFFSET " << ASTSampleRatio::toString(*sample_offset_ratio);
|
||||
}
|
||||
|
||||
return buffer.str();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -58,6 +58,9 @@ public:
|
||||
/// Update tree hash
|
||||
void updateTreeHash(SipHash & hash_state) const;
|
||||
|
||||
/// Format for error message
|
||||
String formatForErrorMessage() const;
|
||||
|
||||
private:
|
||||
bool has_final = false;
|
||||
std::optional<Rational> sample_size_ratio;
|
||||
|
||||
@ -30,11 +30,18 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int TYPE_MISMATCH;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
UnionNode::UnionNode()
|
||||
UnionNode::UnionNode(SelectUnionMode union_mode_)
|
||||
: IQueryTreeNode(children_size)
|
||||
, union_mode(union_mode_)
|
||||
{
|
||||
if (union_mode == SelectUnionMode::UNION_DEFAULT ||
|
||||
union_mode == SelectUnionMode::EXCEPT_DEFAULT ||
|
||||
union_mode == SelectUnionMode::INTERSECT_DEFAULT)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "UNION mode {} must be normalized", toString(union_mode));
|
||||
|
||||
children[queries_child_index] = std::make_shared<ListNode>();
|
||||
}
|
||||
|
||||
@ -101,28 +108,8 @@ void UnionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s
|
||||
buffer << ", constant_value_type: " << constant_value->getType()->getName();
|
||||
}
|
||||
|
||||
if (table_expression_modifiers)
|
||||
{
|
||||
buffer << ", ";
|
||||
table_expression_modifiers->dump(buffer);
|
||||
}
|
||||
|
||||
buffer << ", union_mode: " << toString(union_mode);
|
||||
|
||||
size_t union_modes_size = union_modes.size();
|
||||
buffer << '\n' << std::string(indent + 2, ' ') << "UNION MODES " << union_modes_size << '\n';
|
||||
|
||||
for (size_t i = 0; i < union_modes_size; ++i)
|
||||
{
|
||||
buffer << std::string(indent + 4, ' ');
|
||||
|
||||
auto query_union_mode = union_modes[i];
|
||||
buffer << toString(query_union_mode);
|
||||
|
||||
if (i + 1 != union_modes_size)
|
||||
buffer << '\n';
|
||||
}
|
||||
|
||||
buffer << '\n' << std::string(indent + 2, ' ') << "QUERIES\n";
|
||||
getQueriesNode()->dumpTreeImpl(buffer, format_state, indent + 4);
|
||||
}
|
||||
@ -137,15 +124,8 @@ bool UnionNode::isEqualImpl(const IQueryTreeNode & rhs) const
|
||||
else if (!constant_value && rhs_typed.constant_value)
|
||||
return false;
|
||||
|
||||
if (table_expression_modifiers && rhs_typed.table_expression_modifiers && table_expression_modifiers != rhs_typed.table_expression_modifiers)
|
||||
return false;
|
||||
else if (table_expression_modifiers && !rhs_typed.table_expression_modifiers)
|
||||
return false;
|
||||
else if (!table_expression_modifiers && rhs_typed.table_expression_modifiers)
|
||||
return false;
|
||||
|
||||
return is_subquery == rhs_typed.is_subquery && is_cte == rhs_typed.is_cte && cte_name == rhs_typed.cte_name &&
|
||||
union_mode == rhs_typed.union_mode && union_modes == rhs_typed.union_modes;
|
||||
union_mode == rhs_typed.union_mode;
|
||||
}
|
||||
|
||||
void UnionNode::updateTreeHashImpl(HashState & state) const
|
||||
@ -158,10 +138,6 @@ void UnionNode::updateTreeHashImpl(HashState & state) const
|
||||
|
||||
state.update(static_cast<size_t>(union_mode));
|
||||
|
||||
state.update(union_modes.size());
|
||||
for (const auto & query_union_mode : union_modes)
|
||||
state.update(static_cast<size_t>(query_union_mode));
|
||||
|
||||
if (constant_value)
|
||||
{
|
||||
auto constant_dump = applyVisitor(FieldVisitorToString(), constant_value->getValue());
|
||||
@ -172,23 +148,16 @@ void UnionNode::updateTreeHashImpl(HashState & state) const
|
||||
state.update(constant_value_type_name.size());
|
||||
state.update(constant_value_type_name);
|
||||
}
|
||||
|
||||
if (table_expression_modifiers)
|
||||
table_expression_modifiers->updateTreeHash(state);
|
||||
}
|
||||
|
||||
QueryTreeNodePtr UnionNode::cloneImpl() const
|
||||
{
|
||||
auto result_union_node = std::make_shared<UnionNode>();
|
||||
auto result_union_node = std::make_shared<UnionNode>(union_mode);
|
||||
|
||||
result_union_node->is_subquery = is_subquery;
|
||||
result_union_node->is_cte = is_cte;
|
||||
result_union_node->cte_name = cte_name;
|
||||
result_union_node->union_mode = union_mode;
|
||||
result_union_node->union_modes = union_modes;
|
||||
result_union_node->union_modes_set = union_modes_set;
|
||||
result_union_node->constant_value = constant_value;
|
||||
result_union_node->table_expression_modifiers = table_expression_modifiers;
|
||||
|
||||
return result_union_node;
|
||||
}
|
||||
@ -197,14 +166,7 @@ ASTPtr UnionNode::toASTImpl() const
|
||||
{
|
||||
auto select_with_union_query = std::make_shared<ASTSelectWithUnionQuery>();
|
||||
select_with_union_query->union_mode = union_mode;
|
||||
|
||||
if (union_mode != SelectUnionMode::UNION_DEFAULT &&
|
||||
union_mode != SelectUnionMode::EXCEPT_DEFAULT &&
|
||||
union_mode != SelectUnionMode::INTERSECT_DEFAULT)
|
||||
select_with_union_query->is_normalized = true;
|
||||
|
||||
select_with_union_query->list_of_modes = union_modes;
|
||||
select_with_union_query->set_of_modes = union_modes_set;
|
||||
select_with_union_query->is_normalized = true;
|
||||
select_with_union_query->children.push_back(getQueriesNode()->toAST());
|
||||
select_with_union_query->list_of_selects = select_with_union_query->children.back();
|
||||
|
||||
|
||||
@ -19,6 +19,7 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
/** Union node represents union of queries in query tree.
|
||||
* Union node must be initialized with normalized union mode.
|
||||
*
|
||||
* Example: (SELECT id FROM test_table) UNION ALL (SELECT id FROM test_table_2);
|
||||
* Example: (SELECT id FROM test_table) UNION DISTINCT (SELECT id FROM test_table_2);
|
||||
@ -41,7 +42,8 @@ using UnionNodePtr = std::shared_ptr<UnionNode>;
|
||||
class UnionNode final : public IQueryTreeNode
|
||||
{
|
||||
public:
|
||||
explicit UnionNode();
|
||||
/// Construct union node with normalized union mode
|
||||
explicit UnionNode(SelectUnionMode union_mode_);
|
||||
|
||||
/// Returns true if union node is subquery, false otherwise
|
||||
bool isSubquery() const
|
||||
@ -85,25 +87,6 @@ public:
|
||||
return union_mode;
|
||||
}
|
||||
|
||||
/// Set union mode value
|
||||
void setUnionMode(SelectUnionMode union_mode_value)
|
||||
{
|
||||
union_mode = union_mode_value;
|
||||
}
|
||||
|
||||
/// Get union modes
|
||||
const SelectUnionModes & getUnionModes() const
|
||||
{
|
||||
return union_modes;
|
||||
}
|
||||
|
||||
/// Set union modes value
|
||||
void setUnionModes(const SelectUnionModes & union_modes_value)
|
||||
{
|
||||
union_modes = union_modes_value;
|
||||
union_modes_set = SelectUnionModesSet(union_modes.begin(), union_modes.end());
|
||||
}
|
||||
|
||||
/// Get union node queries
|
||||
const ListNode & getQueries() const
|
||||
{
|
||||
@ -128,24 +111,6 @@ public:
|
||||
return children[queries_child_index];
|
||||
}
|
||||
|
||||
/// Return true if union node has table expression modifiers, false otherwise
|
||||
bool hasTableExpressionModifiers() const
|
||||
{
|
||||
return table_expression_modifiers.has_value();
|
||||
}
|
||||
|
||||
/// Get table expression modifiers
|
||||
const std::optional<TableExpressionModifiers> & getTableExpressionModifiers() const
|
||||
{
|
||||
return table_expression_modifiers;
|
||||
}
|
||||
|
||||
/// Set table expression modifiers
|
||||
void setTableExpressionModifiers(TableExpressionModifiers table_expression_modifiers_value)
|
||||
{
|
||||
table_expression_modifiers = std::move(table_expression_modifiers_value);
|
||||
}
|
||||
|
||||
/// Compute union node projection columns
|
||||
NamesAndTypes computeProjectionColumns() const;
|
||||
|
||||
@ -189,10 +154,7 @@ private:
|
||||
bool is_cte = false;
|
||||
std::string cte_name;
|
||||
SelectUnionMode union_mode;
|
||||
SelectUnionModes union_modes;
|
||||
SelectUnionModesSet union_modes_set;
|
||||
ConstantValuePtr constant_value;
|
||||
std::optional<TableExpressionModifiers> table_expression_modifiers;
|
||||
|
||||
static constexpr size_t queries_child_index = 0;
|
||||
static constexpr size_t children_size = queries_child_index + 1;
|
||||
|
||||
@ -98,11 +98,6 @@ static ASTPtr convertIntoTableExpressionAST(const QueryTreeNodePtr & table_expre
|
||||
|
||||
if (node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION)
|
||||
{
|
||||
if (auto * query_node = table_expression_node->as<QueryNode>())
|
||||
table_expression_modifiers = query_node->getTableExpressionModifiers();
|
||||
else if (auto * union_node = table_expression_node->as<UnionNode>())
|
||||
table_expression_modifiers = union_node->getTableExpressionModifiers();
|
||||
|
||||
result_table_expression->subquery = result_table_expression->children.back();
|
||||
}
|
||||
else if (node_type == QueryTreeNodeType::TABLE || node_type == QueryTreeNodeType::IDENTIFIER)
|
||||
|
||||
@ -176,6 +176,9 @@ public:
|
||||
|
||||
void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override;
|
||||
|
||||
void finalize() override { data->finalize(); }
|
||||
bool isFinalized() const override { return data->isFinalized(); }
|
||||
|
||||
bool isCollationSupported() const override { return getData().isCollationSupported(); }
|
||||
|
||||
size_t getNumberOfDimensions() const;
|
||||
|
||||
@ -93,6 +93,8 @@ public:
|
||||
bool structureEquals(const IColumn & rhs) const override;
|
||||
double getRatioOfDefaultRows(double sample_ratio) const override;
|
||||
void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override;
|
||||
void finalize() override { nested->finalize(); }
|
||||
bool isFinalized() const override { return nested->isFinalized(); }
|
||||
|
||||
const ColumnArray & getNestedColumn() const { return assert_cast<const ColumnArray &>(*nested); }
|
||||
ColumnArray & getNestedColumn() { return assert_cast<ColumnArray &>(*nested); }
|
||||
|
||||
@ -732,8 +732,8 @@ void ColumnObject::get(size_t n, Field & res) const
|
||||
{
|
||||
assert(n < size());
|
||||
res = Object();
|
||||
|
||||
auto & object = res.get<Object &>();
|
||||
|
||||
for (const auto & entry : subcolumns)
|
||||
{
|
||||
auto it = object.try_emplace(entry->path.getPath()).first;
|
||||
@ -744,7 +744,6 @@ void ColumnObject::get(size_t n, Field & res) const
|
||||
void ColumnObject::insertFrom(const IColumn & src, size_t n)
|
||||
{
|
||||
insert(src[n]);
|
||||
finalize();
|
||||
}
|
||||
|
||||
void ColumnObject::insertRangeFrom(const IColumn & src, size_t start, size_t length)
|
||||
@ -792,9 +791,8 @@ MutableColumnPtr ColumnObject::applyForSubcolumns(Func && func) const
|
||||
{
|
||||
if (!isFinalized())
|
||||
{
|
||||
auto finalized = IColumn::mutate(getPtr());
|
||||
auto finalized = cloneFinalized();
|
||||
auto & finalized_object = assert_cast<ColumnObject &>(*finalized);
|
||||
finalized_object.finalize();
|
||||
return finalized_object.applyForSubcolumns(std::forward<Func>(func));
|
||||
}
|
||||
|
||||
|
||||
@ -198,10 +198,6 @@ public:
|
||||
Subcolumns & getSubcolumns() { return subcolumns; }
|
||||
PathsInData getKeys() const;
|
||||
|
||||
/// Finalizes all subcolumns.
|
||||
void finalize();
|
||||
bool isFinalized() const;
|
||||
|
||||
/// Part of interface
|
||||
|
||||
const char * getFamilyName() const override { return "Object"; }
|
||||
@ -219,12 +215,17 @@ public:
|
||||
void popBack(size_t length) override;
|
||||
Field operator[](size_t n) const override;
|
||||
void get(size_t n, Field & res) const override;
|
||||
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
|
||||
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
MutableColumnPtr cloneResized(size_t new_size) const override;
|
||||
|
||||
/// Finalizes all subcolumns.
|
||||
void finalize() override;
|
||||
bool isFinalized() const override;
|
||||
|
||||
/// Order of rows in ColumnObject is undefined.
|
||||
void getPermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation & res) const override;
|
||||
void compareColumn(const IColumn & rhs, size_t rhs_row_num,
|
||||
@ -264,9 +265,7 @@ private:
|
||||
template <typename Func>
|
||||
MutableColumnPtr applyForSubcolumns(Func && func) const;
|
||||
|
||||
/// For given subcolumn return subcolumn from the same Nested type.
|
||||
/// It's used to get shared sized of Nested to insert correct default values.
|
||||
const Subcolumns::Node * getLeafOfTheSameNested(const Subcolumns::NodePtr & entry) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -570,4 +570,15 @@ void ColumnTuple::getIndicesOfNonDefaultRows(Offsets & indices, size_t from, siz
|
||||
return getIndicesOfNonDefaultRowsImpl<ColumnTuple>(indices, from, limit);
|
||||
}
|
||||
|
||||
void ColumnTuple::finalize()
|
||||
{
|
||||
for (auto & column : columns)
|
||||
column->finalize();
|
||||
}
|
||||
|
||||
bool ColumnTuple::isFinalized() const
|
||||
{
|
||||
return std::all_of(columns.begin(), columns.end(), [](const auto & column) { return column->isFinalized(); });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -103,6 +103,8 @@ public:
|
||||
ColumnPtr compress() const override;
|
||||
double getRatioOfDefaultRows(double sample_ratio) const override;
|
||||
void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override;
|
||||
void finalize() override;
|
||||
bool isFinalized() const override;
|
||||
|
||||
size_t tupleSize() const { return columns.size(); }
|
||||
|
||||
|
||||
@ -85,8 +85,8 @@ public:
|
||||
[[nodiscard]] virtual MutablePtr cloneEmpty() const { return cloneResized(0); }
|
||||
|
||||
/// Creates column with the same type and specified size.
|
||||
/// If size is less current size, then data is cut.
|
||||
/// If size is greater, than default values are appended.
|
||||
/// If size is less than current size, then data is cut.
|
||||
/// If size is greater, then default values are appended.
|
||||
[[nodiscard]] virtual MutablePtr cloneResized(size_t /*size*/) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot cloneResized() column {}", getName()); }
|
||||
|
||||
/// Returns number of values in column.
|
||||
@ -453,6 +453,16 @@ public:
|
||||
return getPtr();
|
||||
}
|
||||
|
||||
/// Some columns may require finalization before using of other operations.
|
||||
virtual void finalize() {}
|
||||
virtual bool isFinalized() const { return true; }
|
||||
|
||||
MutablePtr cloneFinalized() const
|
||||
{
|
||||
auto finalized = IColumn::mutate(getPtr());
|
||||
finalized->finalize();
|
||||
return finalized;
|
||||
}
|
||||
|
||||
[[nodiscard]] static MutablePtr mutate(Ptr ptr)
|
||||
{
|
||||
|
||||
@ -128,7 +128,7 @@ namespace DB
|
||||
int number_of_fds_ready = 0;
|
||||
if (!readable_sockets.empty())
|
||||
{
|
||||
number_of_fds_ready = poll(readable_sockets.data(), readable_sockets.size(), static_cast<int>(timeout));
|
||||
number_of_fds_ready = poll(readable_sockets.data(), static_cast<nfds_t>(readable_sockets.size()), static_cast<int>(timeout));
|
||||
}
|
||||
|
||||
if (number_of_fds_ready > 0)
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <base/types.h>
|
||||
#include <base/defines.h>
|
||||
#include "ElementTypes.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -25,6 +26,7 @@ struct DummyJSONParser
|
||||
{
|
||||
public:
|
||||
Element() = default;
|
||||
static ElementType type() { return ElementType::NULL_VALUE; }
|
||||
static bool isInt64() { return false; }
|
||||
static bool isUInt64() { return false; }
|
||||
static bool isDouble() { return false; }
|
||||
|
||||
17
src/Common/JSONParsers/ElementTypes.h
Normal file
17
src/Common/JSONParsers/ElementTypes.h
Normal file
@ -0,0 +1,17 @@
|
||||
#pragma once
|
||||
|
||||
namespace DB
|
||||
{
|
||||
// Enum values match simdjson's for fast conversion
|
||||
enum class ElementType
|
||||
{
|
||||
ARRAY = '[',
|
||||
OBJECT = '{',
|
||||
INT64 = 'l',
|
||||
UINT64 = 'u',
|
||||
DOUBLE = 'd',
|
||||
STRING = '"',
|
||||
BOOL = 't',
|
||||
NULL_VALUE = 'n'
|
||||
};
|
||||
}
|
||||
@ -6,7 +6,7 @@
|
||||
# include <base/types.h>
|
||||
# include <base/defines.h>
|
||||
# include <rapidjson/document.h>
|
||||
|
||||
# include "ElementTypes.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -26,6 +26,20 @@ struct RapidJSONParser
|
||||
ALWAYS_INLINE Element() = default;
|
||||
ALWAYS_INLINE Element(const rapidjson::Value & value_) : ptr(&value_) {} /// NOLINT
|
||||
|
||||
ALWAYS_INLINE ElementType type() const
|
||||
{
|
||||
switch (ptr->GetType())
|
||||
{
|
||||
case rapidjson::kNumberType: return ptr->IsDouble() ? ElementType::DOUBLE : (ptr->IsUint64() ? ElementType::UINT64 : ElementType::INT64);
|
||||
case rapidjson::kStringType: return ElementType::STRING;
|
||||
case rapidjson::kArrayType: return ElementType::ARRAY;
|
||||
case rapidjson::kObjectType: return ElementType::OBJECT;
|
||||
case rapidjson::kTrueType: return ElementType::BOOL;
|
||||
case rapidjson::kFalseType: return ElementType::BOOL;
|
||||
case rapidjson::kNullType: return ElementType::NULL_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool isInt64() const { return ptr->IsInt64(); }
|
||||
ALWAYS_INLINE bool isUInt64() const { return ptr->IsUint64(); }
|
||||
ALWAYS_INLINE bool isDouble() const { return ptr->IsDouble(); }
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
# include <Common/Exception.h>
|
||||
# include <base/defines.h>
|
||||
# include <simdjson.h>
|
||||
|
||||
# include "ElementTypes.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -31,6 +31,21 @@ struct SimdJSONParser
|
||||
ALWAYS_INLINE Element() {} /// NOLINT
|
||||
ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) {} /// NOLINT
|
||||
|
||||
ALWAYS_INLINE ElementType type() const
|
||||
{
|
||||
switch (element.type())
|
||||
{
|
||||
case simdjson::dom::element_type::INT64: return ElementType::INT64;
|
||||
case simdjson::dom::element_type::UINT64: return ElementType::UINT64;
|
||||
case simdjson::dom::element_type::DOUBLE: return ElementType::DOUBLE;
|
||||
case simdjson::dom::element_type::STRING: return ElementType::STRING;
|
||||
case simdjson::dom::element_type::ARRAY: return ElementType::ARRAY;
|
||||
case simdjson::dom::element_type::OBJECT: return ElementType::OBJECT;
|
||||
case simdjson::dom::element_type::BOOL: return ElementType::BOOL;
|
||||
case simdjson::dom::element_type::NULL_VALUE: return ElementType::NULL_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool isInt64() const { return element.type() == simdjson::dom::element_type::INT64; }
|
||||
ALWAYS_INLINE bool isUInt64() const { return element.type() == simdjson::dom::element_type::UINT64; }
|
||||
ALWAYS_INLINE bool isDouble() const { return element.type() == simdjson::dom::element_type::DOUBLE; }
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/thread_local_rng.h>
|
||||
#include <Common/OvercommitTracker.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
#include "config.h"
|
||||
@ -86,6 +87,8 @@ inline std::string_view toDescription(OvercommitResult result)
|
||||
namespace ProfileEvents
|
||||
{
|
||||
extern const Event QueryMemoryLimitExceeded;
|
||||
extern const Event MemoryAllocatorPurge;
|
||||
extern const Event MemoryAllocatorPurgeTimeMicroseconds;
|
||||
}
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
@ -229,7 +232,10 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT
|
||||
{
|
||||
if (free_memory_in_allocator_arenas.exchange(-current_free_memory_in_allocator_arenas) > 0)
|
||||
{
|
||||
Stopwatch watch;
|
||||
mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0);
|
||||
ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge);
|
||||
ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds());
|
||||
}
|
||||
}
|
||||
|
||||
@ -432,7 +438,7 @@ void MemoryTracker::reset()
|
||||
|
||||
void MemoryTracker::setRSS(Int64 rss_, Int64 free_memory_in_allocator_arenas_)
|
||||
{
|
||||
Int64 new_amount = rss_; // - free_memory_in_allocator_arenas_;
|
||||
Int64 new_amount = rss_;
|
||||
total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed);
|
||||
free_memory_in_allocator_arenas.store(free_memory_in_allocator_arenas_, std::memory_order_relaxed);
|
||||
|
||||
|
||||
@ -229,6 +229,8 @@ The server successfully detected this situation and will download merged part fr
|
||||
M(UserTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in user space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \
|
||||
M(SystemTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in OS kernel space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \
|
||||
M(MemoryOvercommitWaitTimeMicroseconds, "Total time spent in waiting for memory to be freed in OvercommitTracker.") \
|
||||
M(MemoryAllocatorPurge, "Total number of times memory allocator purge was requested") \
|
||||
M(MemoryAllocatorPurgeTimeMicroseconds, "Total number of times memory allocator purge was requested") \
|
||||
M(SoftPageFaults, "The number of soft page faults in query execution threads. Soft page fault usually means a miss in the memory allocator cache which required a new memory mapping from the OS and subsequent allocation of a page of physical memory.") \
|
||||
M(HardPageFaults, "The number of hard page faults in query execution threads. High values indicate either that you forgot to turn off swap on your server, or eviction of memory pages of the ClickHouse binary during very high memory pressure, or successful usage of the 'mmap' read method for the tables data.") \
|
||||
\
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <cmath>
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
@ -21,63 +20,56 @@ namespace ErrorCodes
|
||||
/// Just 10^9.
|
||||
static constexpr auto NS = 1000000000UL;
|
||||
|
||||
/// Tracking window. Actually the size is not really important. We just want to avoid
|
||||
/// throttles when there are no actions for a long period time.
|
||||
static const double window_ns = 1ULL * NS;
|
||||
static const size_t default_burst_seconds = 1;
|
||||
|
||||
Throttler::Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent_)
|
||||
: max_speed(max_speed_)
|
||||
, max_burst(max_speed_ * default_burst_seconds)
|
||||
, limit_exceeded_exception_message("")
|
||||
, tokens(max_burst)
|
||||
, parent(parent_)
|
||||
{}
|
||||
|
||||
Throttler::Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_,
|
||||
const std::shared_ptr<Throttler> & parent_)
|
||||
: max_speed(max_speed_)
|
||||
, max_burst(max_speed_ * default_burst_seconds)
|
||||
, limit(limit_)
|
||||
, limit_exceeded_exception_message(limit_exceeded_exception_message_)
|
||||
, tokens(max_burst)
|
||||
, parent(parent_)
|
||||
{}
|
||||
|
||||
void Throttler::add(size_t amount)
|
||||
{
|
||||
size_t new_count;
|
||||
/// This outer variable is always equal to smoothed_speed.
|
||||
/// We use to avoid race condition.
|
||||
double current_speed = 0;
|
||||
|
||||
// Values obtained under lock to be checked after release
|
||||
size_t count_value;
|
||||
double tokens_value;
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
auto now = clock_gettime_ns_adjusted(prev_ns);
|
||||
/// If prev_ns is equal to zero (first `add` call) we known nothing about speed
|
||||
/// and don't track anything.
|
||||
if (max_speed && prev_ns != 0)
|
||||
if (max_speed)
|
||||
{
|
||||
/// Time spent to process the amount of bytes
|
||||
double time_spent = now - prev_ns;
|
||||
|
||||
/// The speed in bytes per second is equal to amount / time_spent in seconds
|
||||
auto new_speed = amount / (time_spent / NS);
|
||||
|
||||
/// We want to make old values of speed less important for our smoothed value
|
||||
/// so we decay it's value with coef.
|
||||
auto decay_coeff = std::pow(0.5, time_spent / window_ns);
|
||||
|
||||
/// Weighted average between previous and new speed
|
||||
smoothed_speed = smoothed_speed * decay_coeff + (1 - decay_coeff) * new_speed;
|
||||
current_speed = smoothed_speed;
|
||||
double delta_seconds = prev_ns ? static_cast<double>(now - prev_ns) / NS : 0;
|
||||
tokens = std::min<double>(tokens + max_speed * delta_seconds - amount, max_burst);
|
||||
}
|
||||
|
||||
count += amount;
|
||||
new_count = count;
|
||||
count_value = count;
|
||||
tokens_value = tokens;
|
||||
prev_ns = now;
|
||||
}
|
||||
|
||||
if (limit && new_count > limit)
|
||||
if (limit && count_value > limit)
|
||||
throw Exception(limit_exceeded_exception_message + std::string(" Maximum: ") + toString(limit), ErrorCodes::LIMIT_EXCEEDED);
|
||||
|
||||
if (max_speed && current_speed > max_speed)
|
||||
/// Wait unless there is positive amount of tokens - throttling
|
||||
if (max_speed && tokens_value < 0)
|
||||
{
|
||||
/// If we was too fast then we have to sleep until our smoothed speed became <= max_speed
|
||||
int64_t sleep_time = static_cast<int64_t>(-window_ns * std::log2(max_speed / current_speed));
|
||||
|
||||
if (sleep_time > 0)
|
||||
{
|
||||
accumulated_sleep += sleep_time;
|
||||
|
||||
sleepForNanoseconds(sleep_time);
|
||||
|
||||
accumulated_sleep -= sleep_time;
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_time / 1000UL);
|
||||
}
|
||||
int64_t sleep_time = static_cast<int64_t>(-tokens_value / max_speed * NS);
|
||||
accumulated_sleep += sleep_time;
|
||||
sleepForNanoseconds(sleep_time);
|
||||
accumulated_sleep -= sleep_time;
|
||||
ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_time / 1000UL);
|
||||
}
|
||||
|
||||
if (parent)
|
||||
@ -89,9 +81,9 @@ void Throttler::reset()
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
count = 0;
|
||||
accumulated_sleep = 0;
|
||||
smoothed_speed = 0;
|
||||
tokens = max_burst;
|
||||
prev_ns = 0;
|
||||
// NOTE: do not zero `accumulated_sleep` to avoid races
|
||||
}
|
||||
|
||||
bool Throttler::isThrottling() const
|
||||
|
||||
@ -10,25 +10,26 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Allows you to limit the speed of something (in entities per second) using sleep.
|
||||
* Specifics of work:
|
||||
* Tracks exponentially (pow of 1/2) smoothed speed with hardcoded window.
|
||||
* See more comments in .cpp file.
|
||||
*
|
||||
* Also allows you to set a limit on the maximum number of entities. If exceeded, an exception will be thrown.
|
||||
/** Allows you to limit the speed of something (in tokens per second) using sleep.
|
||||
* Implemented using Token Bucket Throttling algorithm.
|
||||
* Also allows you to set a limit on the maximum number of tokens. If exceeded, an exception will be thrown.
|
||||
*/
|
||||
class Throttler
|
||||
{
|
||||
public:
|
||||
explicit Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent_ = nullptr)
|
||||
: max_speed(max_speed_), limit_exceeded_exception_message(""), parent(parent_) {}
|
||||
Throttler(size_t max_speed_, size_t max_burst_, const std::shared_ptr<Throttler> & parent_ = nullptr)
|
||||
: max_speed(max_speed_), max_burst(max_burst_), limit_exceeded_exception_message(""), tokens(max_burst), parent(parent_) {}
|
||||
|
||||
explicit Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent_ = nullptr);
|
||||
|
||||
Throttler(size_t max_speed_, size_t max_burst_, size_t limit_, const char * limit_exceeded_exception_message_,
|
||||
const std::shared_ptr<Throttler> & parent_ = nullptr)
|
||||
: max_speed(max_speed_), max_burst(max_burst_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_), tokens(max_burst), parent(parent_) {}
|
||||
|
||||
Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_,
|
||||
const std::shared_ptr<Throttler> & parent_ = nullptr)
|
||||
: max_speed(max_speed_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_), parent(parent_) {}
|
||||
const std::shared_ptr<Throttler> & parent_ = nullptr);
|
||||
|
||||
/// Calculates the smoothed speed, sleeps if required and throws exception on
|
||||
/// limit overflow.
|
||||
/// Use `amount` tokens, sleeps if required or throws exception on limit overflow.
|
||||
void add(size_t amount);
|
||||
|
||||
/// Not thread safe
|
||||
@ -45,15 +46,14 @@ public:
|
||||
|
||||
private:
|
||||
size_t count{0};
|
||||
const size_t max_speed{0};
|
||||
const uint64_t limit{0}; /// 0 - not limited.
|
||||
const size_t max_speed{0}; /// in tokens per second.
|
||||
const size_t max_burst{0}; /// in tokens.
|
||||
const uint64_t limit{0}; /// 0 - not limited.
|
||||
const char * limit_exceeded_exception_message = nullptr;
|
||||
std::mutex mutex;
|
||||
std::atomic<uint64_t> accumulated_sleep{0};
|
||||
/// Smoothed value of current speed. Updated in `add` method.
|
||||
double smoothed_speed{0};
|
||||
/// previous `add` call time (in nanoseconds)
|
||||
uint64_t prev_ns{0};
|
||||
std::atomic<uint64_t> accumulated_sleep{0}; // Accumulated sleep time over all waiting threads
|
||||
double tokens{0}; /// Amount of tokens available in token bucket. Updated in `add` method.
|
||||
uint64_t prev_ns{0}; /// Previous `add` call time (in nanoseconds).
|
||||
|
||||
/// Used to implement a hierarchy of throttlers
|
||||
std::shared_ptr<Throttler> parent;
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <algorithm>
|
||||
#include <utility>
|
||||
#include <base/range.h>
|
||||
#include <base/unaligned.h>
|
||||
#include <Common/hex.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
|
||||
@ -55,8 +56,11 @@ inline bool parseIPv4(const char * src, unsigned char * dst)
|
||||
}
|
||||
if (*(src - 1) != '\0')
|
||||
return false;
|
||||
|
||||
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
reverseMemcpy(dst, &result, sizeof(result));
|
||||
#else
|
||||
memcpy(dst, &result, sizeof(result));
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -48,6 +48,7 @@ public:
|
||||
bool textCanContainOnlyValidUTF8() const override { return nested->textCanContainOnlyValidUTF8(); }
|
||||
bool isComparable() const override { return nested->isComparable(); }
|
||||
bool canBeComparedWithCollation() const override { return nested->canBeComparedWithCollation(); }
|
||||
bool hasDynamicSubcolumns() const override { return nested->hasDynamicSubcolumns(); }
|
||||
|
||||
bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override
|
||||
{
|
||||
|
||||
@ -22,6 +22,27 @@ namespace ErrorCodes
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
DataTypeMap::DataTypeMap(const DataTypePtr & nested_)
|
||||
: nested(nested_)
|
||||
{
|
||||
const auto * type_array = typeid_cast<const DataTypeArray *>(nested.get());
|
||||
if (!type_array)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Expected Array(Tuple(key, value)) type, got {}", nested->getName());
|
||||
|
||||
const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type_array->getNestedType().get());
|
||||
if (!type_tuple)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Expected Array(Tuple(key, value)) type, got {}", nested->getName());
|
||||
|
||||
if (type_tuple->getElements().size() != 2)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Expected Array(Tuple(key, value)) type, got {}", nested->getName());
|
||||
|
||||
key_type = type_tuple->getElement(0);
|
||||
value_type = type_tuple->getElement(1);
|
||||
assertKeyType();
|
||||
}
|
||||
|
||||
DataTypeMap::DataTypeMap(const DataTypes & elems_)
|
||||
{
|
||||
|
||||
@ -23,6 +23,7 @@ private:
|
||||
public:
|
||||
static constexpr bool is_parametric = true;
|
||||
|
||||
explicit DataTypeMap(const DataTypePtr & nested_);
|
||||
explicit DataTypeMap(const DataTypes & elems);
|
||||
DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & value_type_);
|
||||
|
||||
@ -40,6 +41,7 @@ public:
|
||||
bool isComparable() const override { return key_type->isComparable() && value_type->isComparable(); }
|
||||
bool isParametric() const override { return true; }
|
||||
bool haveSubtypes() const override { return true; }
|
||||
bool hasDynamicSubcolumns() const override { return nested->hasDynamicSubcolumns(); }
|
||||
|
||||
const DataTypePtr & getKeyType() const { return key_type; }
|
||||
const DataTypePtr & getValueType() const { return value_type; }
|
||||
|
||||
@ -36,6 +36,7 @@ public:
|
||||
bool haveSubtypes() const override { return false; }
|
||||
bool equals(const IDataType & rhs) const override;
|
||||
bool isParametric() const override { return true; }
|
||||
bool hasDynamicSubcolumns() const override { return true; }
|
||||
|
||||
SerializationPtr doGetDefaultSerialization() const override;
|
||||
|
||||
|
||||
@ -247,6 +247,11 @@ bool DataTypeTuple::haveMaximumSizeOfValue() const
|
||||
return std::all_of(elems.begin(), elems.end(), [](auto && elem) { return elem->haveMaximumSizeOfValue(); });
|
||||
}
|
||||
|
||||
bool DataTypeTuple::hasDynamicSubcolumns() const
|
||||
{
|
||||
return std::any_of(elems.begin(), elems.end(), [](auto && elem) { return elem->hasDynamicSubcolumns(); });
|
||||
}
|
||||
|
||||
bool DataTypeTuple::isComparable() const
|
||||
{
|
||||
return std::all_of(elems.begin(), elems.end(), [](auto && elem) { return elem->isComparable(); });
|
||||
|
||||
@ -50,6 +50,7 @@ public:
|
||||
bool isComparable() const override;
|
||||
bool textCanContainOnlyValidUTF8() const override;
|
||||
bool haveMaximumSizeOfValue() const override;
|
||||
bool hasDynamicSubcolumns() const override;
|
||||
size_t getMaximumSizeOfValueInMemory() const override;
|
||||
size_t getSizeOfValueInMemory() const override;
|
||||
|
||||
|
||||
@ -291,6 +291,9 @@ public:
|
||||
/// Strings, Numbers, Date, DateTime, Nullable
|
||||
virtual bool canBeInsideLowCardinality() const { return false; }
|
||||
|
||||
/// Object, Array(Object), Tuple(..., Object, ...)
|
||||
virtual bool hasDynamicSubcolumns() const { return false; }
|
||||
|
||||
/// Updates avg_value_size_hint for newly read column. Uses to optimize deserialization. Zero expected for first column.
|
||||
static void updateAvgValueSizeHint(const IColumn & column, double & avg_value_size_hint);
|
||||
|
||||
|
||||
@ -1,17 +1,19 @@
|
||||
#include <Storages/StorageSnapshot.h>
|
||||
#include <DataTypes/ObjectUtils.h>
|
||||
#include <DataTypes/DataTypeObject.h>
|
||||
#include <DataTypes/DataTypeNothing.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeMap.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeNested.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <DataTypes/getLeastSupertype.h>
|
||||
#include <DataTypes/NestedUtils.h>
|
||||
#include <Storages/StorageSnapshot.h>
|
||||
#include <Columns/ColumnObject.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnMap.h>
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
#include <Parsers/ASTExpressionList.h>
|
||||
@ -105,10 +107,11 @@ Array createEmptyArrayField(size_t num_dimensions)
|
||||
DataTypePtr getDataTypeByColumn(const IColumn & column)
|
||||
{
|
||||
auto idx = column.getDataType();
|
||||
if (WhichDataType(idx).isSimple())
|
||||
WhichDataType which(idx);
|
||||
if (which.isSimple())
|
||||
return DataTypeFactory::instance().get(String(magic_enum::enum_name(idx)));
|
||||
|
||||
if (WhichDataType(idx).isNothing())
|
||||
if (which.isNothing())
|
||||
return std::make_shared<DataTypeNothing>();
|
||||
|
||||
if (const auto * column_array = checkAndGetColumn<ColumnArray>(&column))
|
||||
@ -132,41 +135,124 @@ static auto extractVector(const std::vector<Tuple> & vec)
|
||||
return res;
|
||||
}
|
||||
|
||||
void convertObjectsToTuples(Block & block, const NamesAndTypesList & extended_storage_columns)
|
||||
static DataTypePtr recreateTupleWithElements(const DataTypeTuple & type_tuple, const DataTypes & elements)
|
||||
{
|
||||
std::unordered_map<String, DataTypePtr> storage_columns_map;
|
||||
for (const auto & [name, type] : extended_storage_columns)
|
||||
storage_columns_map[name] = type;
|
||||
|
||||
for (auto & column : block)
|
||||
{
|
||||
if (!isObject(column.type))
|
||||
continue;
|
||||
|
||||
const auto & column_object = assert_cast<const ColumnObject &>(*column.column);
|
||||
if (!column_object.isFinalized())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot convert to tuple column '{}' from type {}. Column should be finalized first",
|
||||
column.name, column.type->getName());
|
||||
|
||||
std::tie(column.column, column.type) = unflattenObjectToTuple(column_object);
|
||||
|
||||
auto it = storage_columns_map.find(column.name);
|
||||
if (it == storage_columns_map.end())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Column '{}' not found in storage", column.name);
|
||||
|
||||
/// Check that constructed Tuple type and type in storage are compatible.
|
||||
getLeastCommonTypeForObject({column.type, it->second}, true);
|
||||
}
|
||||
return type_tuple.haveExplicitNames()
|
||||
? std::make_shared<DataTypeTuple>(elements, type_tuple.getElementNames())
|
||||
: std::make_shared<DataTypeTuple>(elements);
|
||||
}
|
||||
|
||||
void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block)
|
||||
static std::pair<ColumnPtr, DataTypePtr> convertObjectColumnToTuple(
|
||||
const ColumnObject & column_object, const DataTypeObject & type_object)
|
||||
{
|
||||
if (!storage_snapshot->object_columns.empty())
|
||||
if (!column_object.isFinalized())
|
||||
{
|
||||
auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects();
|
||||
auto storage_columns = storage_snapshot->getColumns(options);
|
||||
convertObjectsToTuples(block, storage_columns);
|
||||
auto finalized = column_object.cloneFinalized();
|
||||
const auto & finalized_object = assert_cast<const ColumnObject &>(*finalized);
|
||||
return convertObjectColumnToTuple(finalized_object, type_object);
|
||||
}
|
||||
|
||||
const auto & subcolumns = column_object.getSubcolumns();
|
||||
|
||||
PathsInData tuple_paths;
|
||||
DataTypes tuple_types;
|
||||
Columns tuple_columns;
|
||||
|
||||
for (const auto & entry : subcolumns)
|
||||
{
|
||||
tuple_paths.emplace_back(entry->path);
|
||||
tuple_types.emplace_back(entry->data.getLeastCommonType());
|
||||
tuple_columns.emplace_back(entry->data.getFinalizedColumnPtr());
|
||||
}
|
||||
|
||||
return unflattenTuple(tuple_paths, tuple_types, tuple_columns);
|
||||
}
|
||||
|
||||
static std::pair<ColumnPtr, DataTypePtr> recursivlyConvertDynamicColumnToTuple(
|
||||
const ColumnPtr & column, const DataTypePtr & type)
|
||||
{
|
||||
if (!type->hasDynamicSubcolumns())
|
||||
return {column, type};
|
||||
|
||||
if (const auto * type_object = typeid_cast<const DataTypeObject *>(type.get()))
|
||||
{
|
||||
const auto & column_object = assert_cast<const ColumnObject &>(*column);
|
||||
return convertObjectColumnToTuple(column_object, *type_object);
|
||||
}
|
||||
|
||||
if (const auto * type_array = typeid_cast<const DataTypeArray *>(type.get()))
|
||||
{
|
||||
const auto & column_array = assert_cast<const ColumnArray &>(*column);
|
||||
auto [new_column, new_type] = recursivlyConvertDynamicColumnToTuple(
|
||||
column_array.getDataPtr(), type_array->getNestedType());
|
||||
|
||||
return
|
||||
{
|
||||
ColumnArray::create(new_column, column_array.getOffsetsPtr()),
|
||||
std::make_shared<DataTypeArray>(std::move(new_type)),
|
||||
};
|
||||
}
|
||||
|
||||
if (const auto * type_map = typeid_cast<const DataTypeMap *>(type.get()))
|
||||
{
|
||||
const auto & column_map = assert_cast<const ColumnMap &>(*column);
|
||||
auto [new_column, new_type] = recursivlyConvertDynamicColumnToTuple(
|
||||
column_map.getNestedColumnPtr(), type_map->getNestedType());
|
||||
|
||||
return
|
||||
{
|
||||
ColumnMap::create(new_column),
|
||||
std::make_shared<DataTypeMap>(std::move(new_type)),
|
||||
};
|
||||
}
|
||||
|
||||
if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type.get()))
|
||||
{
|
||||
const auto & tuple_columns = assert_cast<const ColumnTuple &>(*column).getColumns();
|
||||
const auto & tuple_types = type_tuple->getElements();
|
||||
|
||||
assert(tuple_columns.size() == tuple_types.size());
|
||||
const size_t tuple_size = tuple_types.size();
|
||||
|
||||
Columns new_tuple_columns(tuple_size);
|
||||
DataTypes new_tuple_types(tuple_size);
|
||||
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
{
|
||||
std::tie(new_tuple_columns[i], new_tuple_types[i])
|
||||
= recursivlyConvertDynamicColumnToTuple(tuple_columns[i], tuple_types[i]);
|
||||
}
|
||||
|
||||
return
|
||||
{
|
||||
ColumnTuple::create(new_tuple_columns),
|
||||
recreateTupleWithElements(*type_tuple, new_tuple_types)
|
||||
};
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Type {} unexpectedly has dynamic columns", type->getName());
|
||||
}
|
||||
|
||||
void convertDynamicColumnsToTuples(Block & block, const StorageSnapshotPtr & storage_snapshot)
|
||||
{
|
||||
for (auto & column : block)
|
||||
{
|
||||
if (!column.type->hasDynamicSubcolumns())
|
||||
continue;
|
||||
|
||||
std::tie(column.column, column.type)
|
||||
= recursivlyConvertDynamicColumnToTuple(column.column, column.type);
|
||||
|
||||
GetColumnsOptions options(GetColumnsOptions::AllPhysical);
|
||||
auto storage_column = storage_snapshot->tryGetColumn(options, column.name);
|
||||
if (!storage_column)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Column '{}' not found in storage", column.name);
|
||||
|
||||
auto storage_column_concrete = storage_snapshot->getColumn(options.withExtendedObjects(), column.name);
|
||||
|
||||
/// Check that constructed Tuple type and type in storage are compatible.
|
||||
getLeastCommonTypeForDynamicColumns(
|
||||
storage_column->type, {column.type, storage_column_concrete.type}, true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -217,24 +303,8 @@ void checkObjectHasNoAmbiguosPaths(const PathsInData & paths)
|
||||
}
|
||||
}
|
||||
|
||||
DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambiguos_paths)
|
||||
static DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambiguos_paths)
|
||||
{
|
||||
if (types.empty())
|
||||
return nullptr;
|
||||
|
||||
bool all_equal = true;
|
||||
for (size_t i = 1; i < types.size(); ++i)
|
||||
{
|
||||
if (!types[i]->equals(*types[0]))
|
||||
{
|
||||
all_equal = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (all_equal)
|
||||
return types[0];
|
||||
|
||||
/// Types of subcolumns by path from all tuples.
|
||||
std::unordered_map<PathInData, DataTypes, PathInData::Hash> subcolumns_types;
|
||||
|
||||
@ -287,19 +357,139 @@ DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambi
|
||||
return unflattenTuple(tuple_paths, tuple_types);
|
||||
}
|
||||
|
||||
NameSet getNamesOfObjectColumns(const NamesAndTypesList & columns_list)
|
||||
{
|
||||
NameSet res;
|
||||
for (const auto & [name, type] : columns_list)
|
||||
if (isObject(type))
|
||||
res.insert(name);
|
||||
static DataTypePtr getLeastCommonTypeForDynamicColumnsImpl(
|
||||
const DataTypePtr & type_in_storage, const DataTypes & concrete_types, bool check_ambiguos_paths);
|
||||
|
||||
return res;
|
||||
template<typename Type>
|
||||
static DataTypePtr getLeastCommonTypeForColumnWithNestedType(
|
||||
const Type & type, const DataTypes & concrete_types, bool check_ambiguos_paths)
|
||||
{
|
||||
DataTypes nested_types;
|
||||
nested_types.reserve(concrete_types.size());
|
||||
|
||||
for (const auto & concrete_type : concrete_types)
|
||||
{
|
||||
const auto * type_with_nested_conctete = typeid_cast<const Type *>(concrete_type.get());
|
||||
if (!type_with_nested_conctete)
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH, "Expected {} type, got {}", demangle(typeid(Type).name()), concrete_type->getName());
|
||||
|
||||
nested_types.push_back(type_with_nested_conctete->getNestedType());
|
||||
}
|
||||
|
||||
return std::make_shared<Type>(
|
||||
getLeastCommonTypeForDynamicColumnsImpl(
|
||||
type.getNestedType(), nested_types, check_ambiguos_paths));
|
||||
}
|
||||
|
||||
bool hasObjectColumns(const ColumnsDescription & columns)
|
||||
static DataTypePtr getLeastCommonTypeForTuple(
|
||||
const DataTypeTuple & type, const DataTypes & concrete_types, bool check_ambiguos_paths)
|
||||
{
|
||||
return std::any_of(columns.begin(), columns.end(), [](const auto & column) { return isObject(column.type); });
|
||||
const auto & element_types = type.getElements();
|
||||
DataTypes new_element_types(element_types.size());
|
||||
|
||||
for (size_t i = 0; i < element_types.size(); ++i)
|
||||
{
|
||||
DataTypes concrete_element_types;
|
||||
concrete_element_types.reserve(concrete_types.size());
|
||||
|
||||
for (const auto & type_concrete : concrete_types)
|
||||
{
|
||||
const auto * type_tuple_conctete = typeid_cast<const DataTypeTuple *>(type_concrete.get());
|
||||
if (!type_tuple_conctete)
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH, "Expected Tuple type, got {}", type_concrete->getName());
|
||||
|
||||
concrete_element_types.push_back(type_tuple_conctete->getElement(i));
|
||||
}
|
||||
|
||||
new_element_types[i] = getLeastCommonTypeForDynamicColumnsImpl(
|
||||
element_types[i], concrete_element_types, check_ambiguos_paths);
|
||||
}
|
||||
|
||||
return recreateTupleWithElements(type, new_element_types);
|
||||
}
|
||||
|
||||
static DataTypePtr getLeastCommonTypeForDynamicColumnsImpl(
|
||||
const DataTypePtr & type_in_storage, const DataTypes & concrete_types, bool check_ambiguos_paths)
|
||||
{
|
||||
if (!type_in_storage->hasDynamicSubcolumns())
|
||||
return type_in_storage;
|
||||
|
||||
if (isObject(type_in_storage))
|
||||
return getLeastCommonTypeForObject(concrete_types, check_ambiguos_paths);
|
||||
|
||||
if (const auto * type_array = typeid_cast<const DataTypeArray *>(type_in_storage.get()))
|
||||
return getLeastCommonTypeForColumnWithNestedType(*type_array, concrete_types, check_ambiguos_paths);
|
||||
|
||||
if (const auto * type_map = typeid_cast<const DataTypeMap *>(type_in_storage.get()))
|
||||
return getLeastCommonTypeForColumnWithNestedType(*type_map, concrete_types, check_ambiguos_paths);
|
||||
|
||||
if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type_in_storage.get()))
|
||||
return getLeastCommonTypeForTuple(*type_tuple, concrete_types, check_ambiguos_paths);
|
||||
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Type {} unexpectedly has dynamic columns", type_in_storage->getName());
|
||||
}
|
||||
|
||||
DataTypePtr getLeastCommonTypeForDynamicColumns(
|
||||
const DataTypePtr & type_in_storage, const DataTypes & concrete_types, bool check_ambiguos_paths)
|
||||
{
|
||||
if (concrete_types.empty())
|
||||
return nullptr;
|
||||
|
||||
bool all_equal = true;
|
||||
for (size_t i = 1; i < concrete_types.size(); ++i)
|
||||
{
|
||||
if (!concrete_types[i]->equals(*concrete_types[0]))
|
||||
{
|
||||
all_equal = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (all_equal)
|
||||
return concrete_types[0];
|
||||
|
||||
return getLeastCommonTypeForDynamicColumnsImpl(type_in_storage, concrete_types, check_ambiguos_paths);
|
||||
}
|
||||
|
||||
DataTypePtr createConcreteEmptyDynamicColumn(const DataTypePtr & type_in_storage)
|
||||
{
|
||||
if (!type_in_storage->hasDynamicSubcolumns())
|
||||
return type_in_storage;
|
||||
|
||||
if (isObject(type_in_storage))
|
||||
return std::make_shared<DataTypeTuple>(
|
||||
DataTypes{std::make_shared<DataTypeUInt8>()}, Names{ColumnObject::COLUMN_NAME_DUMMY});
|
||||
|
||||
if (const auto * type_array = typeid_cast<const DataTypeArray *>(type_in_storage.get()))
|
||||
return std::make_shared<DataTypeArray>(
|
||||
createConcreteEmptyDynamicColumn(type_array->getNestedType()));
|
||||
|
||||
if (const auto * type_map = typeid_cast<const DataTypeMap *>(type_in_storage.get()))
|
||||
return std::make_shared<DataTypeMap>(
|
||||
createConcreteEmptyDynamicColumn(type_map->getNestedType()));
|
||||
|
||||
if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type_in_storage.get()))
|
||||
{
|
||||
const auto & elements = type_tuple->getElements();
|
||||
DataTypes new_elements;
|
||||
new_elements.reserve(elements.size());
|
||||
|
||||
for (const auto & element : elements)
|
||||
new_elements.push_back(createConcreteEmptyDynamicColumn(element));
|
||||
|
||||
return recreateTupleWithElements(*type_tuple, new_elements);
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Type {} unexpectedly has dynamic columns", type_in_storage->getName());
|
||||
}
|
||||
|
||||
bool hasDynamicSubcolumns(const ColumnsDescription & columns)
|
||||
{
|
||||
return std::any_of(columns.begin(), columns.end(),
|
||||
[](const auto & column)
|
||||
{
|
||||
return column.type->hasDynamicSubcolumns();
|
||||
});
|
||||
}
|
||||
|
||||
void extendObjectColumns(NamesAndTypesList & columns_list, const ColumnsDescription & object_columns, bool with_subcolumns)
|
||||
@ -320,16 +510,20 @@ void extendObjectColumns(NamesAndTypesList & columns_list, const ColumnsDescript
|
||||
columns_list.splice(columns_list.end(), std::move(subcolumns_list));
|
||||
}
|
||||
|
||||
void updateObjectColumns(ColumnsDescription & object_columns, const NamesAndTypesList & new_columns)
|
||||
void updateObjectColumns(
|
||||
ColumnsDescription & object_columns,
|
||||
const ColumnsDescription & storage_columns,
|
||||
const NamesAndTypesList & new_columns)
|
||||
{
|
||||
for (const auto & new_column : new_columns)
|
||||
{
|
||||
auto object_column = object_columns.tryGetColumn(GetColumnsOptions::All, new_column.name);
|
||||
if (object_column && !object_column->type->equals(*new_column.type))
|
||||
{
|
||||
auto storage_column = storage_columns.getColumn(GetColumnsOptions::All, new_column.name);
|
||||
object_columns.modify(new_column.name, [&](auto & column)
|
||||
{
|
||||
column.type = getLeastCommonTypeForObject({object_column->type, new_column.type});
|
||||
column.type = getLeastCommonTypeForDynamicColumns(storage_column.type, {object_column->type, new_column.type});
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -745,13 +939,6 @@ void replaceMissedSubcolumnsByConstants(
|
||||
addConstantToWithClause(query, name, type);
|
||||
}
|
||||
|
||||
void finalizeObjectColumns(const MutableColumns & columns)
|
||||
{
|
||||
for (const auto & column : columns)
|
||||
if (auto * column_object = typeid_cast<ColumnObject *>(column.get()))
|
||||
column_object->finalize();
|
||||
}
|
||||
|
||||
Field FieldVisitorReplaceScalars::operator()(const Array & x) const
|
||||
{
|
||||
if (num_dimensions_to_keep == 0)
|
||||
@ -768,11 +955,13 @@ size_t FieldVisitorToNumberOfDimensions::operator()(const Array & x)
|
||||
{
|
||||
const size_t size = x.size();
|
||||
size_t dimensions = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
size_t element_dimensions = applyVisitor(*this, x[i]);
|
||||
if (i > 0 && element_dimensions != dimensions)
|
||||
need_fold_dimension = true;
|
||||
|
||||
dimensions = std::max(dimensions, element_dimensions);
|
||||
}
|
||||
|
||||
@ -783,12 +972,13 @@ Field FieldVisitorFoldDimension::operator()(const Array & x) const
|
||||
{
|
||||
if (num_dimensions_to_fold == 0)
|
||||
return x;
|
||||
|
||||
const size_t size = x.size();
|
||||
Array res(size);
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
res[i] = applyVisitor(FieldVisitorFoldDimension(num_dimensions_to_fold - 1), x[i]);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -39,27 +39,31 @@ Array createEmptyArrayField(size_t num_dimensions);
|
||||
DataTypePtr getDataTypeByColumn(const IColumn & column);
|
||||
|
||||
/// Converts Object types and columns to Tuples in @columns_list and @block
|
||||
/// and checks that types are consistent with types in @extended_storage_columns.
|
||||
void convertObjectsToTuples(Block & block, const NamesAndTypesList & extended_storage_columns);
|
||||
void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block);
|
||||
/// and checks that types are consistent with types in @storage_snapshot.
|
||||
void convertDynamicColumnsToTuples(Block & block, const StorageSnapshotPtr & storage_snapshot);
|
||||
|
||||
/// Checks that each path is not the prefix of any other path.
|
||||
void checkObjectHasNoAmbiguosPaths(const PathsInData & paths);
|
||||
|
||||
/// Receives several Tuple types and deduces the least common type among them.
|
||||
DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambiguos_paths = false);
|
||||
DataTypePtr getLeastCommonTypeForDynamicColumns(
|
||||
const DataTypePtr & type_in_storage, const DataTypes & types, bool check_ambiguos_paths = false);
|
||||
|
||||
DataTypePtr createConcreteEmptyDynamicColumn(const DataTypePtr & type_in_storage);
|
||||
|
||||
/// Converts types of object columns to tuples in @columns_list
|
||||
/// according to @object_columns and adds all tuple's subcolumns if needed.
|
||||
void extendObjectColumns(NamesAndTypesList & columns_list, const ColumnsDescription & object_columns, bool with_subcolumns);
|
||||
|
||||
NameSet getNamesOfObjectColumns(const NamesAndTypesList & columns_list);
|
||||
bool hasObjectColumns(const ColumnsDescription & columns);
|
||||
void finalizeObjectColumns(const MutableColumns & columns);
|
||||
/// Checks whether @columns contain any column with dynamic subcolumns.
|
||||
bool hasDynamicSubcolumns(const ColumnsDescription & columns);
|
||||
|
||||
/// Updates types of objects in @object_columns inplace
|
||||
/// according to types in new_columns.
|
||||
void updateObjectColumns(ColumnsDescription & object_columns, const NamesAndTypesList & new_columns);
|
||||
void updateObjectColumns(
|
||||
ColumnsDescription & object_columns,
|
||||
const ColumnsDescription & storage_columns,
|
||||
const NamesAndTypesList & new_columns);
|
||||
|
||||
using DataTypeTuplePtr = std::shared_ptr<DataTypeTuple>;
|
||||
|
||||
@ -142,13 +146,15 @@ public:
|
||||
{
|
||||
if (num_dimensions_to_fold == 0)
|
||||
return x;
|
||||
Array res(1,x);
|
||||
|
||||
Array res(1, x);
|
||||
for (size_t i = 1; i < num_dimensions_to_fold; ++i)
|
||||
{
|
||||
Array new_res;
|
||||
new_res.push_back(std::move(res));
|
||||
res = std::move(new_res);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -163,7 +169,7 @@ private:
|
||||
/// columns-like objects from entry to which Iterator points.
|
||||
/// columns-like object should have fields "name" and "type".
|
||||
template <typename Iterator, typename EntryColumnsGetter>
|
||||
ColumnsDescription getObjectColumns(
|
||||
ColumnsDescription getConcreteObjectColumns(
|
||||
Iterator begin, Iterator end,
|
||||
const ColumnsDescription & storage_columns,
|
||||
EntryColumnsGetter && entry_columns_getter)
|
||||
@ -176,14 +182,8 @@ ColumnsDescription getObjectColumns(
|
||||
/// dummy column will be removed.
|
||||
for (const auto & column : storage_columns)
|
||||
{
|
||||
if (isObject(column.type))
|
||||
{
|
||||
auto tuple_type = std::make_shared<DataTypeTuple>(
|
||||
DataTypes{std::make_shared<DataTypeUInt8>()},
|
||||
Names{ColumnObject::COLUMN_NAME_DUMMY});
|
||||
|
||||
types_in_entries[column.name].push_back(std::move(tuple_type));
|
||||
}
|
||||
if (column.type->hasDynamicSubcolumns())
|
||||
types_in_entries[column.name].push_back(createConcreteEmptyDynamicColumn(column.type));
|
||||
}
|
||||
|
||||
for (auto it = begin; it != end; ++it)
|
||||
@ -192,14 +192,17 @@ ColumnsDescription getObjectColumns(
|
||||
for (const auto & column : entry_columns)
|
||||
{
|
||||
auto storage_column = storage_columns.tryGetPhysical(column.name);
|
||||
if (storage_column && isObject(storage_column->type))
|
||||
if (storage_column && storage_column->type->hasDynamicSubcolumns())
|
||||
types_in_entries[column.name].push_back(column.type);
|
||||
}
|
||||
}
|
||||
|
||||
ColumnsDescription res;
|
||||
for (const auto & [name, types] : types_in_entries)
|
||||
res.add({name, getLeastCommonTypeForObject(types)});
|
||||
{
|
||||
auto storage_column = storage_columns.getPhysical(name);
|
||||
res.add({name, getLeastCommonTypeForDynamicColumns(storage_column.type, types)});
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -249,7 +249,9 @@ public:
|
||||
};
|
||||
|
||||
/// Call before serializeBinaryBulkWithMultipleStreams chain to write something before first mark.
|
||||
/// Column may be used only to retrieve the structure.
|
||||
virtual void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & /*column*/,
|
||||
SerializeBinaryBulkSettings & /*settings*/,
|
||||
SerializeBinaryBulkStatePtr & /*state*/) const {}
|
||||
|
||||
|
||||
@ -246,11 +246,13 @@ void SerializationArray::enumerateStreams(
|
||||
}
|
||||
|
||||
void SerializationArray::serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const
|
||||
{
|
||||
settings.path.push_back(Substream::ArrayElements);
|
||||
nested->serializeBinaryBulkStatePrefix(settings, state);
|
||||
const auto & column_array = assert_cast<const ColumnArray &>(column);
|
||||
nested->serializeBinaryBulkStatePrefix(column_array.getData(), settings, state);
|
||||
settings.path.pop_back();
|
||||
}
|
||||
|
||||
|
||||
@ -41,6 +41,7 @@ public:
|
||||
const SubstreamData & data) const override;
|
||||
|
||||
void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const override;
|
||||
|
||||
|
||||
@ -221,6 +221,7 @@ struct DeserializeStateLowCardinality : public ISerialization::DeserializeBinary
|
||||
};
|
||||
|
||||
void SerializationLowCardinality::serializeBinaryBulkStatePrefix(
|
||||
const IColumn & /*column*/,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const
|
||||
{
|
||||
|
||||
@ -23,6 +23,7 @@ public:
|
||||
const SubstreamData & data) const override;
|
||||
|
||||
void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const override;
|
||||
|
||||
|
||||
@ -270,10 +270,11 @@ void SerializationMap::enumerateStreams(
|
||||
}
|
||||
|
||||
void SerializationMap::serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const
|
||||
{
|
||||
nested->serializeBinaryBulkStatePrefix(settings, state);
|
||||
nested->serializeBinaryBulkStatePrefix(extractNestedColumn(column), settings, state);
|
||||
}
|
||||
|
||||
void SerializationMap::serializeBinaryBulkStateSuffix(
|
||||
|
||||
@ -37,6 +37,7 @@ public:
|
||||
const SubstreamData & data) const override;
|
||||
|
||||
void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const override;
|
||||
|
||||
|
||||
@ -17,11 +17,12 @@ void SerializationNamed::enumerateStreams(
|
||||
}
|
||||
|
||||
void SerializationNamed::serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const
|
||||
{
|
||||
addToPath(settings.path);
|
||||
nested_serialization->serializeBinaryBulkStatePrefix(settings, state);
|
||||
nested_serialization->serializeBinaryBulkStatePrefix(column, settings, state);
|
||||
settings.path.pop_back();
|
||||
}
|
||||
|
||||
|
||||
@ -31,6 +31,7 @@ public:
|
||||
const SubstreamData & data) const override;
|
||||
|
||||
void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const override;
|
||||
|
||||
|
||||
@ -70,11 +70,13 @@ void SerializationNullable::enumerateStreams(
|
||||
}
|
||||
|
||||
void SerializationNullable::serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const
|
||||
{
|
||||
settings.path.push_back(Substream::NullableElements);
|
||||
nested->serializeBinaryBulkStatePrefix(settings, state);
|
||||
const auto & column_nullable = assert_cast<const ColumnNullable &>(column);
|
||||
nested->serializeBinaryBulkStatePrefix(column_nullable.getNestedColumn(), settings, state);
|
||||
settings.path.pop_back();
|
||||
}
|
||||
|
||||
|
||||
@ -19,6 +19,7 @@ public:
|
||||
const SubstreamData & data) const override;
|
||||
|
||||
void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const override;
|
||||
|
||||
|
||||
@ -13,8 +13,6 @@
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Functions/FunctionsConversion.h>
|
||||
|
||||
#include <Common/FieldVisitorToString.h>
|
||||
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/VarInt.h>
|
||||
@ -30,6 +28,7 @@ namespace ErrorCodes
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int INCORRECT_DATA;
|
||||
extern const int CANNOT_READ_ALL_DATA;
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
@ -141,7 +140,6 @@ void SerializationObject<Parser>::checkSerializationIsSupported(const TSettings
|
||||
template <typename Parser>
|
||||
struct SerializationObject<Parser>::SerializeStateObject : public ISerialization::SerializeBinaryBulkState
|
||||
{
|
||||
bool is_first = true;
|
||||
DataTypePtr nested_type;
|
||||
SerializationPtr nested_serialization;
|
||||
SerializeBinaryBulkStatePtr nested_state;
|
||||
@ -158,6 +156,7 @@ struct SerializationObject<Parser>::DeserializeStateObject : public ISerializati
|
||||
|
||||
template <typename Parser>
|
||||
void SerializationObject<Parser>::serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const
|
||||
{
|
||||
@ -166,15 +165,34 @@ void SerializationObject<Parser>::serializeBinaryBulkStatePrefix(
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED,
|
||||
"DataTypeObject doesn't support serialization with non-trivial state");
|
||||
|
||||
const auto & column_object = assert_cast<const ColumnObject &>(column);
|
||||
if (!column_object.isFinalized())
|
||||
{
|
||||
auto finalized = column_object.cloneFinalized();
|
||||
serializeBinaryBulkStatePrefix(*finalized, settings, state);
|
||||
return;
|
||||
}
|
||||
|
||||
settings.path.push_back(Substream::ObjectStructure);
|
||||
auto * stream = settings.getter(settings.path);
|
||||
settings.path.pop_back();
|
||||
|
||||
if (!stream)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Missing stream for kind of binary serialization");
|
||||
|
||||
auto [tuple_column, tuple_type] = unflattenObjectToTuple(column_object);
|
||||
|
||||
writeIntBinary(static_cast<UInt8>(BinarySerializationKind::TUPLE), *stream);
|
||||
state = std::make_shared<SerializeStateObject>();
|
||||
writeStringBinary(tuple_type->getName(), *stream);
|
||||
|
||||
auto state_object = std::make_shared<SerializeStateObject>();
|
||||
state_object->nested_type = tuple_type;
|
||||
state_object->nested_serialization = tuple_type->getDefaultSerialization();
|
||||
|
||||
settings.path.back() = Substream::ObjectData;
|
||||
state_object->nested_serialization->serializeBinaryBulkStatePrefix(*tuple_column, settings, state_object->nested_state);
|
||||
|
||||
state = std::move(state_object);
|
||||
settings.path.pop_back();
|
||||
}
|
||||
|
||||
template <typename Parser>
|
||||
@ -261,33 +279,14 @@ void SerializationObject<Parser>::serializeBinaryBulkWithMultipleStreams(
|
||||
|
||||
if (!column_object.isFinalized())
|
||||
{
|
||||
auto finalized_object = column_object.clone();
|
||||
assert_cast<ColumnObject &>(*finalized_object).finalize();
|
||||
serializeBinaryBulkWithMultipleStreams(*finalized_object, offset, limit, settings, state);
|
||||
auto finalized = column_object.cloneFinalized();
|
||||
serializeBinaryBulkWithMultipleStreams(*finalized, offset, limit, settings, state);
|
||||
return;
|
||||
}
|
||||
|
||||
auto [tuple_column, tuple_type] = unflattenObjectToTuple(column_object);
|
||||
|
||||
if (state_object->is_first)
|
||||
{
|
||||
/// Actually it's a part of serializeBinaryBulkStatePrefix,
|
||||
/// but it cannot be done there, because we have to know the
|
||||
/// structure of column.
|
||||
|
||||
settings.path.push_back(Substream::ObjectStructure);
|
||||
if (auto * stream = settings.getter(settings.path))
|
||||
writeStringBinary(tuple_type->getName(), *stream);
|
||||
|
||||
state_object->nested_type = tuple_type;
|
||||
state_object->nested_serialization = tuple_type->getDefaultSerialization();
|
||||
state_object->is_first = false;
|
||||
|
||||
settings.path.back() = Substream::ObjectData;
|
||||
state_object->nested_serialization->serializeBinaryBulkStatePrefix(settings, state_object->nested_state);
|
||||
settings.path.pop_back();
|
||||
}
|
||||
else if (!state_object->nested_type->equals(*tuple_type))
|
||||
if (!state_object->nested_type->equals(*tuple_type))
|
||||
{
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Types of internal column of Object mismatched. Expected: {}, Got: {}",
|
||||
@ -411,18 +410,63 @@ void SerializationObject<Parser>::serializeTextImpl(const IColumn & column, size
|
||||
writeChar('{', ostr);
|
||||
for (auto it = subcolumns.begin(); it != subcolumns.end(); ++it)
|
||||
{
|
||||
const auto & entry = *it;
|
||||
if (it != subcolumns.begin())
|
||||
writeCString(",", ostr);
|
||||
|
||||
writeDoubleQuoted((*it)->path.getPath(), ostr);
|
||||
writeDoubleQuoted(entry->path.getPath(), ostr);
|
||||
writeChar(':', ostr);
|
||||
|
||||
auto serialization = (*it)->data.getLeastCommonType()->getDefaultSerialization();
|
||||
serialization->serializeTextJSON((*it)->data.getFinalizedColumn(), row_num, ostr, settings);
|
||||
serializeTextFromSubcolumn(entry->data, row_num, ostr, settings);
|
||||
}
|
||||
writeChar('}', ostr);
|
||||
}
|
||||
|
||||
template <typename Parser>
|
||||
void SerializationObject<Parser>::serializeTextFromSubcolumn(
|
||||
const ColumnObject::Subcolumn & subcolumn, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
const auto & least_common_type = subcolumn.getLeastCommonType();
|
||||
|
||||
if (subcolumn.isFinalized())
|
||||
{
|
||||
const auto & finalized_column = subcolumn.getFinalizedColumn();
|
||||
auto info = least_common_type->getSerializationInfo(finalized_column);
|
||||
auto serialization = least_common_type->getSerialization(*info);
|
||||
serialization->serializeTextJSON(finalized_column, row_num, ostr, settings);
|
||||
return;
|
||||
}
|
||||
|
||||
size_t ind = row_num;
|
||||
if (ind < subcolumn.getNumberOfDefaultsInPrefix())
|
||||
{
|
||||
/// Suboptimal, but it should happen rarely.
|
||||
auto tmp_column = subcolumn.getLeastCommonType()->createColumn();
|
||||
tmp_column->insertDefault();
|
||||
|
||||
auto info = least_common_type->getSerializationInfo(*tmp_column);
|
||||
auto serialization = least_common_type->getSerialization(*info);
|
||||
serialization->serializeTextJSON(*tmp_column, 0, ostr, settings);
|
||||
return;
|
||||
}
|
||||
|
||||
ind -= subcolumn.getNumberOfDefaultsInPrefix();
|
||||
for (const auto & part : subcolumn.getData())
|
||||
{
|
||||
if (ind < part->size())
|
||||
{
|
||||
auto part_type = getDataTypeByColumn(*part);
|
||||
auto info = part_type->getSerializationInfo(*part);
|
||||
auto serialization = part_type->getSerialization(*info);
|
||||
serialization->serializeTextJSON(*part, ind, ostr, settings);
|
||||
return;
|
||||
}
|
||||
|
||||
ind -= part->size();
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Index ({}) for text serialization is out of range", row_num);
|
||||
}
|
||||
|
||||
template <typename Parser>
|
||||
void SerializationObject<Parser>::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
|
||||
@ -8,7 +8,7 @@ namespace DB
|
||||
{
|
||||
|
||||
/** Serialization for data type Object.
|
||||
* Supported only test serialization/deserialization.
|
||||
* Supported only text serialization/deserialization.
|
||||
* and binary bulk serialization/deserialization without position independent
|
||||
* encoding, i.e. serialization/deserialization into Native format.
|
||||
*/
|
||||
@ -31,6 +31,7 @@ public:
|
||||
*/
|
||||
|
||||
void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const override;
|
||||
|
||||
@ -104,6 +105,7 @@ private:
|
||||
void deserializeTextImpl(IColumn & column, Reader && reader) const;
|
||||
|
||||
void serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const;
|
||||
void serializeTextFromSubcolumn(const ColumnObject::Subcolumn & subcolumn, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const;
|
||||
|
||||
/// Pool of parser objects to make SerializationObject thread safe.
|
||||
mutable SimpleObjectPool<Parser> parsers_pool;
|
||||
|
||||
@ -178,11 +178,16 @@ void SerializationSparse::enumerateStreams(
|
||||
}
|
||||
|
||||
void SerializationSparse::serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const
|
||||
{
|
||||
settings.path.push_back(Substream::SparseElements);
|
||||
nested->serializeBinaryBulkStatePrefix(settings, state);
|
||||
if (const auto * column_sparse = typeid_cast<const ColumnSparse *>(&column))
|
||||
nested->serializeBinaryBulkStatePrefix(column_sparse->getValuesColumn(), settings, state);
|
||||
else
|
||||
nested->serializeBinaryBulkStatePrefix(column, settings, state);
|
||||
|
||||
settings.path.pop_back();
|
||||
}
|
||||
|
||||
|
||||
@ -33,6 +33,7 @@ public:
|
||||
const SubstreamData & data) const override;
|
||||
|
||||
void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const override;
|
||||
|
||||
|
||||
@ -314,6 +314,7 @@ struct DeserializeBinaryBulkStateTuple : public ISerialization::DeserializeBinar
|
||||
|
||||
|
||||
void SerializationTuple::serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const
|
||||
{
|
||||
@ -321,7 +322,7 @@ void SerializationTuple::serializeBinaryBulkStatePrefix(
|
||||
tuple_state->states.resize(elems.size());
|
||||
|
||||
for (size_t i = 0; i < elems.size(); ++i)
|
||||
elems[i]->serializeBinaryBulkStatePrefix(settings, tuple_state->states[i]);
|
||||
elems[i]->serializeBinaryBulkStatePrefix(extractElementColumn(column, i), settings, tuple_state->states[i]);
|
||||
|
||||
state = std::move(tuple_state);
|
||||
}
|
||||
|
||||
@ -39,6 +39,7 @@ public:
|
||||
const SubstreamData & data) const override;
|
||||
|
||||
void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const override;
|
||||
|
||||
|
||||
@ -13,10 +13,11 @@ void SerializationWrapper::enumerateStreams(
|
||||
}
|
||||
|
||||
void SerializationWrapper::serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const
|
||||
{
|
||||
nested_serialization->serializeBinaryBulkStatePrefix(settings, state);
|
||||
nested_serialization->serializeBinaryBulkStatePrefix(column, settings, state);
|
||||
}
|
||||
|
||||
void SerializationWrapper::serializeBinaryBulkStateSuffix(
|
||||
|
||||
@ -26,6 +26,7 @@ public:
|
||||
const SubstreamData & data) const override;
|
||||
|
||||
void serializeBinaryBulkStatePrefix(
|
||||
const IColumn & column,
|
||||
SerializeBinaryBulkSettings & settings,
|
||||
SerializeBinaryBulkStatePtr & state) const override;
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@ TEST(SerializationObject, FromString)
|
||||
settings.getter = [&out](const auto &) { return &out; };
|
||||
|
||||
writeIntBinary(static_cast<UInt8>(1), out);
|
||||
serialization->serializeBinaryBulkStatePrefix(settings, state);
|
||||
serialization->serializeBinaryBulkStatePrefix(*column_string, settings, state);
|
||||
serialization->serializeBinaryBulkWithMultipleStreams(*column_string, 0, column_string->size(), settings, state);
|
||||
serialization->serializeBinaryBulkStateSuffix(settings, state);
|
||||
}
|
||||
|
||||
@ -141,7 +141,7 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
|
||||
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(finalize_callback), object.absolute_path);
|
||||
}
|
||||
|
||||
void AzureObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
|
||||
void AzureObjectStorage::findAllFiles(const std::string & path, RelativePathsWithSize & children) const
|
||||
{
|
||||
auto client_ptr = client.get();
|
||||
|
||||
|
||||
@ -84,7 +84,7 @@ public:
|
||||
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
|
||||
const WriteSettings & write_settings = {}) override;
|
||||
|
||||
void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
|
||||
void findAllFiles(const std::string & path, RelativePathsWithSize & children) const override;
|
||||
|
||||
/// Remove file. Throws exception if file doesn't exists or it's a directory.
|
||||
void removeObject(const StoredObject & object) override;
|
||||
|
||||
@ -282,9 +282,9 @@ std::unique_ptr<IObjectStorage> CachedObjectStorage::cloneObjectStorage(
|
||||
return object_storage->cloneObjectStorage(new_namespace, config, config_prefix, context);
|
||||
}
|
||||
|
||||
void CachedObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
|
||||
void CachedObjectStorage::findAllFiles(const std::string & path, RelativePathsWithSize & children) const
|
||||
{
|
||||
object_storage->listPrefix(path, children);
|
||||
object_storage->findAllFiles(path, children);
|
||||
}
|
||||
|
||||
ObjectMetadata CachedObjectStorage::getObjectMetadata(const std::string & path) const
|
||||
|
||||
@ -72,7 +72,7 @@ public:
|
||||
const std::string & config_prefix,
|
||||
ContextPtr context) override;
|
||||
|
||||
void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
|
||||
void findAllFiles(const std::string & path, RelativePathsWithSize & children) const override;
|
||||
|
||||
ObjectMetadata getObjectMetadata(const std::string & path) const override;
|
||||
|
||||
|
||||
@ -390,7 +390,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage *
|
||||
};
|
||||
|
||||
RelativePathsWithSize children;
|
||||
source_object_storage->listPrefix(restore_information.source_path, children);
|
||||
source_object_storage->findAllFiles(restore_information.source_path, children);
|
||||
|
||||
restore_files(children);
|
||||
|
||||
@ -540,7 +540,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject
|
||||
};
|
||||
|
||||
RelativePathsWithSize children;
|
||||
source_object_storage->listPrefix(restore_information.source_path + "operations/", children);
|
||||
source_object_storage->findAllFiles(restore_information.source_path + "operations/", children);
|
||||
restore_file_operations(children);
|
||||
|
||||
if (restore_information.detached)
|
||||
|
||||
@ -101,18 +101,6 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
|
||||
}
|
||||
|
||||
|
||||
void HDFSObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
|
||||
{
|
||||
const size_t begin_of_path = path.find('/', path.find("//") + 2);
|
||||
int32_t num_entries;
|
||||
auto * files_list = hdfsListDirectory(hdfs_fs.get(), path.substr(begin_of_path).c_str(), &num_entries);
|
||||
if (num_entries == -1)
|
||||
throw Exception(ErrorCodes::HDFS_ERROR, "HDFSDelete failed with path: " + path);
|
||||
|
||||
for (int32_t i = 0; i < num_entries; ++i)
|
||||
children.emplace_back(files_list[i].mName, files_list[i].mSize);
|
||||
}
|
||||
|
||||
/// Remove file. Throws exception if file doesn't exists or it's a directory.
|
||||
void HDFSObjectStorage::removeObject(const StoredObject & object)
|
||||
{
|
||||
|
||||
@ -85,8 +85,6 @@ public:
|
||||
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
|
||||
const WriteSettings & write_settings = {}) override;
|
||||
|
||||
void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
|
||||
|
||||
/// Remove file. Throws exception if file doesn't exists or it's a directory.
|
||||
void removeObject(const StoredObject & object) override;
|
||||
|
||||
|
||||
@ -11,10 +11,16 @@
|
||||
#include <Disks/DirectoryIterator.h>
|
||||
#include <Disks/WriteMode.h>
|
||||
#include <Disks/ObjectStorages/IObjectStorage.h>
|
||||
#include <Common/ErrorCodes.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
class IMetadataStorage;
|
||||
|
||||
/// Tries to provide some "transactions" interface, which allow
|
||||
@ -33,32 +39,71 @@ public:
|
||||
/// General purpose methods
|
||||
|
||||
/// Write metadata string to file
|
||||
virtual void writeStringToFile(const std::string & path, const std::string & data) = 0;
|
||||
virtual void writeStringToFile(const std::string & /* path */, const std::string & /* data */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void setLastModified(const std::string & path, const Poco::Timestamp & timestamp) = 0;
|
||||
virtual void setLastModified(const std::string & /* path */, const Poco::Timestamp & /* timestamp */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual bool supportsChmod() const = 0;
|
||||
virtual void chmod(const String & path, mode_t mode) = 0;
|
||||
virtual void chmod(const String & /* path */, mode_t /* mode */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void setReadOnly(const std::string & path) = 0;
|
||||
virtual void setReadOnly(const std::string & /* path */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void unlinkFile(const std::string & path) = 0;
|
||||
virtual void unlinkFile(const std::string & /* path */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void createDirectory(const std::string & path) = 0;
|
||||
virtual void createDirectory(const std::string & /* path */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void createDirectoryRecursive(const std::string & path) = 0;
|
||||
virtual void createDirectoryRecursive(const std::string & /* path */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void removeDirectory(const std::string & path) = 0;
|
||||
virtual void removeDirectory(const std::string & /* path */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void removeRecursive(const std::string & path) = 0;
|
||||
virtual void removeRecursive(const std::string & /* path */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void createHardLink(const std::string & path_from, const std::string & path_to) = 0;
|
||||
virtual void createHardLink(const std::string & /* path_from */, const std::string & /* path_to */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void moveFile(const std::string & path_from, const std::string & path_to) = 0;
|
||||
virtual void moveFile(const std::string & /* path_from */, const std::string & /* path_to */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void moveDirectory(const std::string & path_from, const std::string & path_to) = 0;
|
||||
virtual void moveDirectory(const std::string & /* path_from */, const std::string & /* path_to */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual void replaceFile(const std::string & path_from, const std::string & path_to) = 0;
|
||||
virtual void replaceFile(const std::string & /* path_from */, const std::string & /* path_to */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
/// Metadata related methods
|
||||
|
||||
@ -69,7 +114,10 @@ public:
|
||||
virtual void createMetadataFile(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes) = 0;
|
||||
|
||||
/// Add to new blob to metadata file (way to implement appends)
|
||||
virtual void addBlobToMetadata(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes) = 0;
|
||||
virtual void addBlobToMetadata(const std::string & /* path */, const std::string & /* blob_name */, uint64_t /* size_in_bytes */)
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
/// Unlink metadata file and do something special if required
|
||||
/// By default just remove file (unlink file).
|
||||
@ -79,6 +127,12 @@ public:
|
||||
}
|
||||
|
||||
virtual ~IMetadataTransaction() = default;
|
||||
|
||||
private:
|
||||
[[noreturn]] static void throwNotImplemented()
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Operation is not implemented");
|
||||
}
|
||||
};
|
||||
|
||||
using MetadataTransactionPtr = std::shared_ptr<IMetadataTransaction>;
|
||||
@ -106,12 +160,18 @@ public:
|
||||
|
||||
virtual Poco::Timestamp getLastModified(const std::string & path) const = 0;
|
||||
|
||||
virtual time_t getLastChanged(const std::string & path) const = 0;
|
||||
virtual time_t getLastChanged(const std::string & /* path */) const
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual bool supportsChmod() const = 0;
|
||||
|
||||
virtual bool supportsStat() const = 0;
|
||||
virtual struct stat stat(const String & path) const = 0;
|
||||
virtual struct stat stat(const String & /* path */) const
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual std::vector<std::string> listDirectory(const std::string & path) const = 0;
|
||||
|
||||
@ -120,20 +180,32 @@ public:
|
||||
virtual uint32_t getHardlinkCount(const std::string & path) const = 0;
|
||||
|
||||
/// Read metadata file to string from path
|
||||
virtual std::string readFileToString(const std::string & path) const = 0;
|
||||
virtual std::string readFileToString(const std::string & /* path */) const
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
virtual ~IMetadataStorage() = default;
|
||||
|
||||
/// ==== More specific methods. Previous were almost general purpose. ====
|
||||
|
||||
/// Read multiple metadata files into strings and return mapping from file_path -> metadata
|
||||
virtual std::unordered_map<std::string, std::string> getSerializedMetadata(const std::vector<String> & file_paths) const = 0;
|
||||
virtual std::unordered_map<std::string, std::string> getSerializedMetadata(const std::vector<String> & /* file_paths */) const
|
||||
{
|
||||
throwNotImplemented();
|
||||
}
|
||||
|
||||
/// Return object information (absolute_path, bytes_size, ...) for metadata path.
|
||||
/// object_storage_path is absolute.
|
||||
virtual StoredObjects getStorageObjects(const std::string & path) const = 0;
|
||||
|
||||
virtual std::string getObjectStorageRootPath() const = 0;
|
||||
|
||||
private:
|
||||
[[noreturn]] static void throwNotImplemented()
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Operation is not implemented");
|
||||
}
|
||||
};
|
||||
|
||||
using MetadataStoragePtr = std::shared_ptr<IMetadataStorage>;
|
||||
|
||||
@ -14,6 +14,17 @@ namespace ErrorCodes
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
void IObjectStorage::findAllFiles(const std::string &, RelativePathsWithSize &) const
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "findAllFiles() is not supported");
|
||||
}
|
||||
void IObjectStorage::getDirectoryContents(const std::string &,
|
||||
RelativePathsWithSize &,
|
||||
std::vector<std::string> &) const
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getDirectoryContents() is not supported");
|
||||
}
|
||||
|
||||
IAsynchronousReader & IObjectStorage::getThreadPoolReader()
|
||||
{
|
||||
auto context = Context::getGlobalContextInstance();
|
||||
|
||||
@ -65,8 +65,32 @@ public:
|
||||
/// Object exists or not
|
||||
virtual bool exists(const StoredObject & object) const = 0;
|
||||
|
||||
/// List on prefix, return children (relative paths) with their sizes.
|
||||
virtual void listPrefix(const std::string & path, RelativePathsWithSize & children) const = 0;
|
||||
/// List all objects with specific prefix.
|
||||
///
|
||||
/// For example if you do this over filesystem, you should skip folders and
|
||||
/// return files only, so something like on local filesystem:
|
||||
///
|
||||
/// find . -type f
|
||||
///
|
||||
/// @param children - out files (relative paths) with their sizes.
|
||||
///
|
||||
/// NOTE: It makes sense only for real object storages (S3, Azure), since
|
||||
/// it is used only for one of the following:
|
||||
/// - send_metadata (to restore metadata)
|
||||
/// - see DiskObjectStorage::restoreMetadataIfNeeded()
|
||||
/// - MetadataStorageFromPlainObjectStorage - only for s3_plain disk
|
||||
virtual void findAllFiles(const std::string & path, RelativePathsWithSize & children) const;
|
||||
|
||||
/// Analog of directory content for object storage (object storage does not
|
||||
/// have "directory" definition, but it can be emulated with usage of
|
||||
/// "delimiter"), so this is analog of:
|
||||
///
|
||||
/// find . -maxdepth 1 $path
|
||||
///
|
||||
/// Return files in @files and directories in @directories
|
||||
virtual void getDirectoryContents(const std::string & path,
|
||||
RelativePathsWithSize & files,
|
||||
std::vector<std::string> & directories) const;
|
||||
|
||||
/// Get object metadata if supported. It should be possible to receive
|
||||
/// at least size of object
|
||||
|
||||
@ -104,13 +104,6 @@ std::unique_ptr<WriteBufferFromFileBase> LocalObjectStorage::writeObject( /// NO
|
||||
return std::make_unique<WriteBufferFromFile>(path, buf_size, flags);
|
||||
}
|
||||
|
||||
void LocalObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
|
||||
{
|
||||
fs::directory_iterator end_it;
|
||||
for (auto it = fs::directory_iterator(path); it != end_it; ++it)
|
||||
children.emplace_back(it->path().filename(), it->file_size());
|
||||
}
|
||||
|
||||
void LocalObjectStorage::removeObject(const StoredObject & object)
|
||||
{
|
||||
/// For local object storage files are actually removed when "metadata" is removed.
|
||||
|
||||
@ -45,8 +45,6 @@ public:
|
||||
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
|
||||
const WriteSettings & write_settings = {}) override;
|
||||
|
||||
void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
|
||||
|
||||
void removeObject(const StoredObject & object) override;
|
||||
|
||||
void removeObjects(const StoredObjects & objects) override;
|
||||
|
||||
@ -12,7 +12,6 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
@ -33,194 +32,102 @@ const std::string & MetadataStorageFromPlainObjectStorage::getPath() const
|
||||
{
|
||||
return object_storage_root_path;
|
||||
}
|
||||
std::filesystem::path MetadataStorageFromPlainObjectStorage::getAbsolutePath(const std::string & path) const
|
||||
{
|
||||
return fs::path(object_storage_root_path) / path;
|
||||
}
|
||||
|
||||
bool MetadataStorageFromPlainObjectStorage::exists(const std::string & path) const
|
||||
{
|
||||
auto object = StoredObject::create(*object_storage, fs::path(object_storage_root_path) / path);
|
||||
auto object = StoredObject::create(*object_storage, getAbsolutePath(path));
|
||||
return object_storage->exists(object);
|
||||
}
|
||||
|
||||
bool MetadataStorageFromPlainObjectStorage::isFile(const std::string & path) const
|
||||
{
|
||||
/// NOTE: This check is inaccurate and has excessive API calls
|
||||
return !isDirectory(path) && exists(path);
|
||||
return exists(path) && !isDirectory(path);
|
||||
}
|
||||
|
||||
bool MetadataStorageFromPlainObjectStorage::isDirectory(const std::string & path) const
|
||||
{
|
||||
std::string directory = path;
|
||||
std::string directory = getAbsolutePath(path);
|
||||
trimRight(directory);
|
||||
directory += "/";
|
||||
|
||||
/// NOTE: This check is far from ideal, since it work only if the directory
|
||||
/// really has files, and has excessive API calls
|
||||
RelativePathsWithSize children;
|
||||
object_storage->listPrefix(directory, children);
|
||||
return !children.empty();
|
||||
}
|
||||
|
||||
Poco::Timestamp MetadataStorageFromPlainObjectStorage::getLastModified(const std::string &) const
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getLastModified is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
struct stat MetadataStorageFromPlainObjectStorage::stat(const std::string &) const
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "stat is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
time_t MetadataStorageFromPlainObjectStorage::getLastChanged(const std::string &) const
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getLastChanged is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
RelativePathsWithSize files;
|
||||
std::vector<std::string> directories;
|
||||
object_storage->getDirectoryContents(directory, files, directories);
|
||||
return !files.empty() || !directories.empty();
|
||||
}
|
||||
|
||||
uint64_t MetadataStorageFromPlainObjectStorage::getFileSize(const String & path) const
|
||||
{
|
||||
RelativePathsWithSize children;
|
||||
object_storage->listPrefix(path, children);
|
||||
object_storage->findAllFiles(getAbsolutePath(path), children);
|
||||
if (children.empty())
|
||||
return 0;
|
||||
if (children.size() != 1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "listPrefix() return multiple paths ({}) for {}", children.size(), path);
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "findAllFiles() return multiple paths ({}) for {}", children.size(), path);
|
||||
return children.front().bytes_size;
|
||||
}
|
||||
|
||||
std::vector<std::string> MetadataStorageFromPlainObjectStorage::listDirectory(const std::string & path) const
|
||||
{
|
||||
RelativePathsWithSize children;
|
||||
object_storage->listPrefix(path, children);
|
||||
RelativePathsWithSize files;
|
||||
std::vector<std::string> directories;
|
||||
object_storage->getDirectoryContents(getAbsolutePath(path), files, directories);
|
||||
|
||||
std::vector<std::string> result;
|
||||
for (const auto & path_size : children)
|
||||
{
|
||||
for (const auto & path_size : files)
|
||||
result.push_back(path_size.relative_path);
|
||||
}
|
||||
for (const auto & directory : directories)
|
||||
result.push_back(directory);
|
||||
return result;
|
||||
}
|
||||
|
||||
DirectoryIteratorPtr MetadataStorageFromPlainObjectStorage::iterateDirectory(const std::string & path) const
|
||||
{
|
||||
/// NOTE: this is not required for BACKUP/RESTORE, but this is a first step
|
||||
/// towards MergeTree on plain S3.
|
||||
/// Required for MergeTree
|
||||
auto paths = listDirectory(path);
|
||||
std::vector<std::filesystem::path> fs_paths(paths.begin(), paths.end());
|
||||
return std::make_unique<StaticDirectoryIterator>(std::move(fs_paths));
|
||||
}
|
||||
|
||||
std::string MetadataStorageFromPlainObjectStorage::readFileToString(const std::string &) const
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "readFileToString is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
std::unordered_map<String, String> MetadataStorageFromPlainObjectStorage::getSerializedMetadata(const std::vector<String> &) const
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getSerializedMetadata is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
StoredObjects MetadataStorageFromPlainObjectStorage::getStorageObjects(const std::string & path) const
|
||||
{
|
||||
std::string blob_name = object_storage->generateBlobNameForPath(path);
|
||||
|
||||
std::string object_path = fs::path(object_storage_root_path) / blob_name;
|
||||
size_t object_size = getFileSize(object_path);
|
||||
|
||||
auto object = StoredObject::create(*object_storage, object_path, object_size, /* exists */true);
|
||||
size_t object_size = getFileSize(blob_name);
|
||||
auto object = StoredObject::create(*object_storage, getAbsolutePath(blob_name), object_size, /* exists */true);
|
||||
return {std::move(object)};
|
||||
}
|
||||
|
||||
uint32_t MetadataStorageFromPlainObjectStorage::getHardlinkCount(const std::string &) const
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
const IMetadataStorage & MetadataStorageFromPlainObjectStorageTransaction::getStorageForNonTransactionalReads() const
|
||||
{
|
||||
return metadata_storage;
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::writeStringToFile(const std::string &, const std::string & /* data */)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "writeStringToFile is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::setLastModified(const std::string &, const Poco::Timestamp &)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "setLastModified is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::unlinkFile(const std::string & path)
|
||||
{
|
||||
auto object = StoredObject::create(*metadata_storage.object_storage, fs::path(metadata_storage.object_storage_root_path) / path);
|
||||
auto object = StoredObject::create(*metadata_storage.object_storage, metadata_storage.getAbsolutePath(path));
|
||||
metadata_storage.object_storage->removeObject(object);
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::removeRecursive(const std::string &)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "removeRecursive is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::createDirectory(const std::string &)
|
||||
{
|
||||
/// Noop. It is an Object Storage not a filesystem.
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::createDirectoryRecursive(const std::string &)
|
||||
{
|
||||
/// Noop. It is an Object Storage not a filesystem.
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::removeDirectory(const std::string &)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "removeDirectory is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::moveFile(const std::string & /* path_from */, const std::string & /* path_to */)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "moveFile is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::moveDirectory(const std::string & /* path_from */, const std::string & /* path_to */)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "moveDirectory is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::replaceFile(const std::string & /* path_from */, const std::string & /* path_to */)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "replaceFile is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::chmod(const String &, mode_t)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "chmod is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::setReadOnly(const std::string &)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "setReadOnly is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::createHardLink(const std::string & /* path_from */, const std::string & /* path_to */)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "createHardLink is not implemented for MetadataStorageFromPlainObjectStorage");
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::createEmptyMetadataFile(const std::string &)
|
||||
{
|
||||
/// Noop, no separate metadata.
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::createMetadataFile(
|
||||
const std::string &, const std::string & /* blob_name */, uint64_t /* size_in_bytes */)
|
||||
{
|
||||
/// Noop, no separate metadata.
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::addBlobToMetadata(
|
||||
const std::string &, const std::string & /* blob_name */, uint64_t /* size_in_bytes */)
|
||||
{
|
||||
/// Noop, local metadata files is only one file, it is the metadata file itself.
|
||||
}
|
||||
|
||||
void MetadataStorageFromPlainObjectStorageTransaction::unlinkMetadata(const std::string &)
|
||||
{
|
||||
/// Noop, no separate metadata.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user