mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Merge remote-tracking branch 'upstream/master' into fix-projection-merge
This commit is contained in:
jsc0218
commit
216f75cd0a
18
.github/actions/clean/action.yml
vendored
18
.github/actions/clean/action.yml
vendored
@ -1,11 +1,23 @@
|
||||
name: Clean runner
|
||||
description: Clean the runner's temp path on ending
|
||||
inputs:
|
||||
images:
|
||||
description: clean docker images
|
||||
default: false
|
||||
type: boolean
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Clean
|
||||
- name: Clean Temp
|
||||
shell: bash
|
||||
run: |
|
||||
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
|
||||
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
|
||||
sudo rm -fr "${{runner.temp}}"
|
||||
- name: Clean Docker Containers
|
||||
shell: bash
|
||||
run: |
|
||||
docker rm -vf $(docker ps -aq) ||:
|
||||
- name: Clean Docker Images
|
||||
if: ${{ inputs.images }}
|
||||
shell: bash
|
||||
run: |
|
||||
docker rmi -f $(docker images -aq) ||:
|
||||
|
||||
82
.github/workflows/auto_releases.yml
vendored
82
.github/workflows/auto_releases.yml
vendored
@ -14,18 +14,16 @@ on:
|
||||
dry-run:
|
||||
description: 'Dry run'
|
||||
required: false
|
||||
default: true
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
AutoReleaseInfo:
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
runs-on: [self-hosted, release-maker]
|
||||
outputs:
|
||||
data: ${{ steps.info.outputs.AUTO_RELEASE_PARAMS }}
|
||||
dry_run: ${{ steps.info.outputs.DRY_RUN }}
|
||||
steps:
|
||||
- name: Debug Info
|
||||
uses: ./.github/actions/debug
|
||||
- name: Set envs
|
||||
run: |
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
@ -36,6 +34,10 @@ jobs:
|
||||
echo "DRY_RUN=true" >> "$GITHUB_ENV"
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
fetch-depth: 0 # full history needed
|
||||
- name: Debug Info
|
||||
uses: ./.github/actions/debug
|
||||
- name: Prepare Info
|
||||
id: info
|
||||
run: |
|
||||
@ -46,15 +48,14 @@ jobs:
|
||||
echo "::endgroup::"
|
||||
{
|
||||
echo 'AUTO_RELEASE_PARAMS<<EOF'
|
||||
cat /tmp/autorelease_info.json
|
||||
echo 'EOF'
|
||||
} >> "$GITHUB_ENV"
|
||||
{
|
||||
echo 'AUTO_RELEASE_PARAMS<<EOF'
|
||||
cat /tmp/autorelease_info.json
|
||||
cat /tmp/autorelease_params.json
|
||||
echo 'EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
if [[ "${{ github.event_name }}" == "schedule" ]]; then
|
||||
echo "DRY_RUN=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "DRY_RUN=${{ github.event.inputs.dry-run }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- name: Post Release Branch statuses
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
@ -62,48 +63,37 @@ jobs:
|
||||
- name: Clean up
|
||||
uses: ./.github/actions/clean
|
||||
|
||||
Release_0:
|
||||
Releases:
|
||||
needs: AutoReleaseInfo
|
||||
name: Release ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0].release_branch }}
|
||||
if: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0] && fromJson(needs.AutoReleaseInfo.outputs.data).releases[0].ready }}
|
||||
strategy:
|
||||
matrix:
|
||||
release_params: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases }}
|
||||
max-parallel: 1
|
||||
name: Release ${{ matrix.release_params.release_branch }}
|
||||
uses: ./.github/workflows/create_release.yml
|
||||
with:
|
||||
ref: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0].commit_sha }}
|
||||
ref: ${{ matrix.release_params.commit_sha }}
|
||||
type: patch
|
||||
dry-run: ${{ needs.AutoReleaseInfo.outputs.dry_run }}
|
||||
#
|
||||
# Release_1:
|
||||
# needs: [AutoReleaseInfo, Release_0]
|
||||
# name: Release ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[1].release_branch }}
|
||||
# if: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[1] && fromJson(needs.AutoReleaseInfo.outputs.data).releases[1].ready }}
|
||||
# uses: ./.github/workflows/create_release.yml
|
||||
# with:
|
||||
# ref: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[1].commit_sha }}
|
||||
# type: patch
|
||||
# dry-run: ${{ env.DRY_RUN }}
|
||||
#
|
||||
# Release_2:
|
||||
# needs: [AutoReleaseInfo, Release_1]
|
||||
# name: Release ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[2].release_branch }}
|
||||
# if: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0] && fromJson(needs.AutoReleaseInfo.outputs.data).releases[2].ready }}
|
||||
# uses: ./.github/workflow/create_release.yml
|
||||
# with:
|
||||
# ref: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[0].commit_sha }}
|
||||
# type: patch
|
||||
# dry-run: ${{ env.DRY_RUN }}
|
||||
#
|
||||
# Release_3:
|
||||
# needs: [AutoReleaseInfo, Release_2]
|
||||
# name: Release ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[3].release_branch }}
|
||||
# if: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[3] && fromJson(needs.AutoReleaseInfo.outputs.data).releases[3].ready }}
|
||||
# uses: ./.github/workflow/create_release.yml
|
||||
# with:
|
||||
# ref: ${{ fromJson(needs.AutoReleaseInfo.outputs.data).releases[3].commit_sha }}
|
||||
# type: patch
|
||||
# dry-run: ${{ env.DRY_RUN }}
|
||||
dry-run: ${{ fromJson(needs.AutoReleaseInfo.outputs.dry_run) }}
|
||||
secrets:
|
||||
ROBOT_CLICKHOUSE_COMMIT_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
|
||||
|
||||
# - name: Post Slack Message
|
||||
CleanUp:
|
||||
needs: [Releases]
|
||||
runs-on: [self-hosted, release-maker]
|
||||
steps:
|
||||
- uses: ./.github/actions/clean
|
||||
with:
|
||||
images: true
|
||||
|
||||
# PostSlackMessage:
|
||||
# needs: [Releases]
|
||||
# runs-on: [self-hosted, release-maker]
|
||||
# if: ${{ !cancelled() }}
|
||||
# steps:
|
||||
# - name: Check out repository code
|
||||
# uses: ClickHouse/checkout@v1
|
||||
# - name: Post
|
||||
# run: |
|
||||
# cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
# python3 auto_release.py --post-auto-release-complete --wf-status ${{ job.status }}
|
||||
|
||||
2
.github/workflows/create_release.yml
vendored
2
.github/workflows/create_release.yml
vendored
@ -47,6 +47,8 @@ concurrency:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
secrets:
|
||||
ROBOT_CLICKHOUSE_COMMIT_TOKEN:
|
||||
|
||||
jobs:
|
||||
CreateRelease:
|
||||
|
||||
10
README.md
10
README.md
@ -34,13 +34,19 @@ curl https://clickhouse.com/ | sh
|
||||
|
||||
Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
|
||||
|
||||
* [v24.8 Community Call](https://clickhouse.com/company/events/v24-8-community-release-call) - August 29
|
||||
* [v24.8 Community Call](https://clickhouse.com/company/events/v24-8-community-release-call) - August 20
|
||||
|
||||
## Upcoming Events
|
||||
|
||||
Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
|
||||
|
||||
* MORE COMING SOON!
|
||||
The following upcoming meetups are featuring creator of ClickHouse & CTO, Alexey Milovidov:
|
||||
|
||||
* [ClickHouse Guangzhou User Group Meetup](https://mp.weixin.qq.com/s/GSvo-7xUoVzCsuUvlLTpCw) - August 25
|
||||
* [San Francisco Meetup (Cloudflare)](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/302540575) - September 5
|
||||
* [Raleigh Meetup (Deutsche Bank)](https://www.meetup.com/clickhouse-nc-meetup-group/events/302557230) - September 9
|
||||
* [New York Meetup (Ramp)](https://www.meetup.com/clickhouse-new-york-user-group/events/302575342) - September 10
|
||||
* [Chicago Meetup (Jump Capital)](https://lu.ma/43tvmrfw) - September 12
|
||||
|
||||
## Recent Recordings
|
||||
* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
|
||||
|
||||
16
SECURITY.md
16
SECURITY.md
@ -14,25 +14,15 @@ The following versions of ClickHouse server are currently supported with securit
|
||||
|
||||
| Version | Supported |
|
||||
|:-|:-|
|
||||
| 24.8 | ✔️ |
|
||||
| 24.7 | ✔️ |
|
||||
| 24.6 | ✔️ |
|
||||
| 24.5 | ✔️ |
|
||||
| 24.5 | ❌ |
|
||||
| 24.4 | ❌ |
|
||||
| 24.3 | ✔️ |
|
||||
| 24.2 | ❌ |
|
||||
| 24.1 | ❌ |
|
||||
| 23.12 | ❌ |
|
||||
| 23.11 | ❌ |
|
||||
| 23.10 | ❌ |
|
||||
| 23.9 | ❌ |
|
||||
| 23.8 | ✔️ |
|
||||
| 23.7 | ❌ |
|
||||
| 23.6 | ❌ |
|
||||
| 23.5 | ❌ |
|
||||
| 23.4 | ❌ |
|
||||
| 23.3 | ❌ |
|
||||
| 23.2 | ❌ |
|
||||
| 23.1 | ❌ |
|
||||
| 23.* | ❌ |
|
||||
| 22.* | ❌ |
|
||||
| 21.* | ❌ |
|
||||
| 20.* | ❌ |
|
||||
|
||||
@ -47,8 +47,7 @@
|
||||
"docker/test/stateful": {
|
||||
"name": "clickhouse/stateful-test",
|
||||
"dependent": [
|
||||
"docker/test/stress",
|
||||
"docker/test/upgrade"
|
||||
"docker/test/stress"
|
||||
]
|
||||
},
|
||||
"docker/test/unit": {
|
||||
@ -59,10 +58,6 @@
|
||||
"name": "clickhouse/stress-test",
|
||||
"dependent": []
|
||||
},
|
||||
"docker/test/upgrade": {
|
||||
"name": "clickhouse/upgrade-check",
|
||||
"dependent": []
|
||||
},
|
||||
"docker/test/integration/runner": {
|
||||
"name": "clickhouse/integration-tests-runner",
|
||||
"dependent": []
|
||||
|
||||
@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.7.3.42"
|
||||
ARG VERSION="24.8.1.2684"
|
||||
ARG PACKAGES="clickhouse-keeper"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.7.3.42"
|
||||
ARG VERSION="24.8.1.2684"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
|
||||
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
|
||||
ARG VERSION="24.7.3.42"
|
||||
ARG VERSION="24.8.1.2684"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
#docker-official-library:off
|
||||
|
||||
@ -93,6 +93,3 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
ENV COMMIT_SHA=''
|
||||
ENV PULL_REQUEST_NUMBER=''
|
||||
ENV COPY_CLICKHOUSE_BINARY_TO_OUTPUT=0
|
||||
|
||||
COPY run.sh /
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
|
||||
@ -10,7 +10,3 @@ RUN apt-get update -y \
|
||||
npm \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
COPY create.sql /
|
||||
COPY run.sh /
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
|
||||
@ -1 +0,0 @@
|
||||
../stateless/setup_minio.sh
|
||||
@ -85,18 +85,6 @@ RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoo
|
||||
ENV MINIO_ROOT_USER="clickhouse"
|
||||
ENV MINIO_ROOT_PASSWORD="clickhouse"
|
||||
ENV EXPORT_S3_STORAGE_POLICIES=1
|
||||
ENV CLICKHOUSE_GRPC_CLIENT="/usr/share/clickhouse-utils/grpc-client/clickhouse-grpc-client.py"
|
||||
|
||||
RUN npm install -g azurite@3.30.0 \
|
||||
&& npm install -g tslib && npm install -g node
|
||||
|
||||
COPY run.sh /
|
||||
COPY setup_minio.sh /
|
||||
COPY setup_hdfs_minicluster.sh /
|
||||
COPY attach_gdb.lib /
|
||||
COPY utils.lib /
|
||||
|
||||
# We store stress_tests.lib in stateless image to avoid duplication of this file in stress and upgrade tests
|
||||
COPY stress_tests.lib /
|
||||
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
|
||||
@ -22,8 +22,5 @@ RUN apt-get update -y \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
COPY run.sh /
|
||||
|
||||
ENV EXPORT_S3_STORAGE_POLICIES=1
|
||||
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
|
||||
@ -1,29 +0,0 @@
|
||||
# rebuild in #33610
|
||||
# docker build -t clickhouse/upgrade-check .
|
||||
ARG FROM_TAG=latest
|
||||
FROM clickhouse/stateful-test:$FROM_TAG
|
||||
|
||||
RUN apt-get update -y \
|
||||
&& env DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get install --yes --no-install-recommends \
|
||||
bash \
|
||||
tzdata \
|
||||
parallel \
|
||||
expect \
|
||||
python3 \
|
||||
python3-lxml \
|
||||
python3-termcolor \
|
||||
python3-requests \
|
||||
curl \
|
||||
sudo \
|
||||
openssl \
|
||||
netcat-openbsd \
|
||||
brotli \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
COPY run.sh /
|
||||
|
||||
ENV EXPORT_S3_STORAGE_POLICIES=1
|
||||
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
@ -56,7 +56,5 @@ RUN apt-get update \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
COPY process_functional_tests_result.py /
|
||||
|
||||
COPY --from=clickhouse/cctools:0d6b90a7a490 /opt/gdb /opt/gdb
|
||||
ENV PATH="/opt/gdb/bin:${PATH}"
|
||||
|
||||
@ -1,38 +0,0 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v23.8.16.16-lts (b80cac57ead) FIXME as compared to v23.8.15.35-lts (060ff8e813a)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#66962](https://github.com/ClickHouse/ClickHouse/issues/66962): Added support for parameterized view with analyzer to not analyze create parameterized view. Refactor existing parameterized view logic to not analyze create parameterized view. [#54211](https://github.com/ClickHouse/ClickHouse/pull/54211) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
|
||||
* Backported in [#65461](https://github.com/ClickHouse/ClickHouse/issues/65461): Reload certificate chain during certificate reload. [#61671](https://github.com/ClickHouse/ClickHouse/pull/61671) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)).
|
||||
* Backported in [#65880](https://github.com/ClickHouse/ClickHouse/issues/65880): Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Backported in [#65912](https://github.com/ClickHouse/ClickHouse/issues/65912): Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#65351](https://github.com/ClickHouse/ClickHouse/issues/65351): Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#66037](https://github.com/ClickHouse/ClickHouse/issues/66037): Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#65281](https://github.com/ClickHouse/ClickHouse/issues/65281): Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#65368](https://github.com/ClickHouse/ClickHouse/issues/65368): Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Backported in [#65782](https://github.com/ClickHouse/ClickHouse/issues/65782): Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Backported in [#65743](https://github.com/ClickHouse/ClickHouse/issues/65743): Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#65926](https://github.com/ClickHouse/ClickHouse/issues/65926): For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Backported in [#65822](https://github.com/ClickHouse/ClickHouse/issues/65822): Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
|
||||
* Backported in [#66322](https://github.com/ClickHouse/ClickHouse/issues/66322): Add missing settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines/input_format_csv_try_infer_numbers_from_strings/input_format_csv_try_infer_strings_from_quoted_tuples` in schema inference cache because they can change the resulting schema. It prevents from incorrect result of schema inference with these settings changed. [#65980](https://github.com/ClickHouse/ClickHouse/pull/65980) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#66449](https://github.com/ClickHouse/ClickHouse/issues/66449): Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#66717](https://github.com/ClickHouse/ClickHouse/issues/66717): Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#67320](https://github.com/ClickHouse/ClickHouse/issues/67320): TRUNCATE DATABASE used to stop replication as if it was a DROP DATABASE query, it's fixed. [#67129](https://github.com/ClickHouse/ClickHouse/pull/67129) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#65080](https://github.com/ClickHouse/ClickHouse/issues/65080): Follow up to [#56541](https://github.com/ClickHouse/ClickHouse/issues/56541). [#57141](https://github.com/ClickHouse/ClickHouse/pull/57141) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#64997](https://github.com/ClickHouse/ClickHouse/issues/64997): Fix crash with DISTINCT and window functions. [#64767](https://github.com/ClickHouse/ClickHouse/pull/64767) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Backported in [#65913](https://github.com/ClickHouse/ClickHouse/issues/65913): Fix bug with session closing in Keeper. [#65735](https://github.com/ClickHouse/ClickHouse/pull/65735) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#66853](https://github.com/ClickHouse/ClickHouse/issues/66853): Fix data race in S3::ClientCache. [#66644](https://github.com/ClickHouse/ClickHouse/pull/66644) ([Konstantin Morozov](https://github.com/k-morozov)).
|
||||
* Backported in [#67072](https://github.com/ClickHouse/ClickHouse/issues/67072): Increase asio pool size in case the server is tiny. [#66761](https://github.com/ClickHouse/ClickHouse/pull/66761) ([alesapin](https://github.com/alesapin)).
|
||||
* Update version after release. [#67691](https://github.com/ClickHouse/ClickHouse/pull/67691) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
|
||||
525
docs/changelogs/v24.8.1.2684-lts.md
Normal file
525
docs/changelogs/v24.8.1.2684-lts.md
Normal file
@ -0,0 +1,525 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.8.1.2684-lts (161c62fd295) FIXME as compared to v24.8.1.1-new (3f8b27d7acc)
|
||||
|
||||
#### Backward Incompatible Change
|
||||
* `clickhouse-client` and `clickhouse-local` now default to multi-query mode (instead single-query mode). As an example, `clickhouse-client -q "SELECT 1; SELECT 2"` now works, whereas users previously had to add `--multiquery` (or `-n`). The `--multiquery/-n` switch became obsolete. INSERT queries in multi-query statements are treated specially based on their FORMAT clause: If the FORMAT is `VALUES` (the most common case), the end of the INSERT statement is represented by a trailing semicolon `;` at the end of the query. For all other FORMATs (e.g. `CSV` or `JSONEachRow`), the end of the INSERT statement is represented by two newlines `\n\n` at the end of the query. [#63898](https://github.com/ClickHouse/ClickHouse/pull/63898) ([FFish](https://github.com/wxybear)).
|
||||
* In previous versions, it was possible to use an alternative syntax for `LowCardinality` data types by appending `WithDictionary` to the name of the data type. It was an initial working implementation, and it was never documented or exposed to the public. Now, it is deprecated. If you have used this syntax, you have to ALTER your tables and rename the data types to `LowCardinality`. [#66842](https://github.com/ClickHouse/ClickHouse/pull/66842) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix logical errors with storage `Buffer` used with distributed destination table. It's a backward incompatible change: queries using `Buffer` with a distributed destination table may stop working if the table appears more than once in the query (e.g., in a self-join). [#67015](https://github.com/ClickHouse/ClickHouse/pull/67015) ([vdimir](https://github.com/vdimir)).
|
||||
* In previous versions, calling functions for random distributions based on the Gamma function (such as Chi-Squared, Student, Fisher) with negative arguments close to zero led to a long computation or an infinite loop. In the new version, calling these functions with zero or negative arguments will produce an exception. This closes [#67297](https://github.com/ClickHouse/ClickHouse/issues/67297). [#67326](https://github.com/ClickHouse/ClickHouse/pull/67326) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* The system table `text_log` is enabled by default. This is fully compatible with previous versions, but you may notice subtly increased disk usage on the local disk (this system table takes a tiny amount of disk space). [#67428](https://github.com/ClickHouse/ClickHouse/pull/67428) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* In previous versions, `arrayWithConstant` can be slow if asked to generate very large arrays. In the new version, it is limited to 1 GB per array. This closes [#32754](https://github.com/ClickHouse/ClickHouse/issues/32754). [#67741](https://github.com/ClickHouse/ClickHouse/pull/67741) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix REPLACE modifier formatting (forbid omitting brackets). [#67774](https://github.com/ClickHouse/ClickHouse/pull/67774) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Backported in [#68349](https://github.com/ClickHouse/ClickHouse/issues/68349): Reimplement Dynamic type. Now when the limit of dynamic data types is reached new types are not casted to String but stored in a special data structure in binary format with binary encoded data type. Now any type ever inserted into Dynamic column can be read from it as subcolumn. [#68132](https://github.com/ClickHouse/ClickHouse/pull/68132) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
#### New Feature
|
||||
* Add new experimental Kafka storage engine to store offsets in Keeper instead of relying on committing them to Kafka. [#57625](https://github.com/ClickHouse/ClickHouse/pull/57625) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Add new TimeSeries table engine: - by default: ``` CREATE TABLE tbl ENGINE=TimeSeries ``` - or with specifying engines of its internal tables:. [#64183](https://github.com/ClickHouse/ClickHouse/pull/64183) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Support more join strictnesses (`LEFT/RIGHT SEMI/ANTI/ANY JOIN`) with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y (see setting `allow_experimental_join_condition`). [#64281](https://github.com/ClickHouse/ClickHouse/pull/64281) ([lgbo](https://github.com/lgbo-ustc)).
|
||||
* Add `_etag` virtual column for S3 table engine. Fixes [#65312](https://github.com/ClickHouse/ClickHouse/issues/65312). [#65386](https://github.com/ClickHouse/ClickHouse/pull/65386) ([skyoct](https://github.com/skyoct)).
|
||||
* This pull request introduces Hive-style partitioning for different engines (`File`, `URL`, `S3`, `AzureBlobStorage`, `HDFS`). Hive-style partitioning organizes data into partitioned sub-directories, making it efficient to query and manage large datasets. Currently, it only creates virtual columns with the appropriate name and data. The follow-up PR will introduce the appropriate data filtering (performance speedup). [#65997](https://github.com/ClickHouse/ClickHouse/pull/65997) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Add function printf for spark compatiability. [#66257](https://github.com/ClickHouse/ClickHouse/pull/66257) ([李扬](https://github.com/taiyang-li)).
|
||||
* Backported in [#68450](https://github.com/ClickHouse/ClickHouse/issues/68450): Implement new JSON data type. [#66444](https://github.com/ClickHouse/ClickHouse/pull/66444) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Add a new server setting: disable_insertion_and_mutation Set it to true. This node will deny all insertions and mutations(Alter table delete/update/drop partition). Include async insertion. [#66519](https://github.com/ClickHouse/ClickHouse/pull/66519) ([Xu Jia](https://github.com/XuJia0210)).
|
||||
* Add options `restore_replace_external_engines_to_null` and `restore_replace_external_table_functions_to_null` to replace external engines and table_engines to Null engine that can be useful for testing. It should work for RESTORE and explicit table creation. [#66536](https://github.com/ClickHouse/ClickHouse/pull/66536) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Added support for reading MULTILINESTRING geometry in WKT format using function readWKTLineString. [#67647](https://github.com/ClickHouse/ClickHouse/pull/67647) ([Jacob Reckhard](https://github.com/jacobrec)).
|
||||
* Add a new table function `fuzzQuery`. This function allows the modification of a given query string with random variations. Example: `SELECT query FROM fuzzQuery('SELECT 1') LIMIT 5;`. [#67655](https://github.com/ClickHouse/ClickHouse/pull/67655) ([pufit](https://github.com/pufit)).
|
||||
* Support query `DROP DETACHED PARTITION ALL` to drop all detached partitions. [#67885](https://github.com/ClickHouse/ClickHouse/pull/67885) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Added a tagging (namespace) mechanism for the query cache. The same queries with different tags are considered different by the query cache. Example: `SELECT 1 SETTINGS use_query_cache = 1, query_cache_tag = 'abc'` and `SELECT 1 SETTINGS use_query_cache = 1, query_cache_tag = 'def'` now create different query cache entries. [#68235](https://github.com/ClickHouse/ClickHouse/pull/68235) ([sakulali](https://github.com/sakulali)).
|
||||
|
||||
#### Performance Improvement
|
||||
* Use adaptive read task size calculation method (adaptive meaning it depends on read column sizes) for parallel replicas. [#60377](https://github.com/ClickHouse/ClickHouse/pull/60377) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Store the `plain_rewritable` disk directory metadata in `__meta` layout, separately from the merge tree data in the object storage. Move the `plain_rewritable` disk to a flat directory structure. [#65751](https://github.com/ClickHouse/ClickHouse/pull/65751) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Enable `compile_expressions` (JIT compiler for fragments of ordinary expressions) by default. This closes [#51264](https://github.com/ClickHouse/ClickHouse/issues/51264) and [#56386](https://github.com/ClickHouse/ClickHouse/issues/56386). [#66486](https://github.com/ClickHouse/ClickHouse/pull/66486) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Improve columns squashing for String/Array/Map/Variant/Dynamic types by reserving required memory in advance for all subcolumns. [#67043](https://github.com/ClickHouse/ClickHouse/pull/67043) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Speed up system flush logs, flush logs on shutdown. [#67472](https://github.com/ClickHouse/ClickHouse/pull/67472) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Backported in [#68496](https://github.com/ClickHouse/ClickHouse/issues/68496): Improved overall performance of merges by reducing the overhead of scheduling steps of merges. [#68016](https://github.com/ClickHouse/ClickHouse/pull/68016) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Setting `optimize_functions_to_subcolumns` is enabled by default. [#68053](https://github.com/ClickHouse/ClickHouse/pull/68053) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Improvement
|
||||
* ClickHouse server now supports new setting `max_keep_alive_requests`. For keep-alive HTTP connections to the server it works in tandem with `keep_alive_timeout` - if idle timeout not expired but there already more than `max_keep_alive_requests` requests done through the given connection - it will be closed by the server. [#61793](https://github.com/ClickHouse/ClickHouse/pull/61793) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* As in the new version, SOURCES are checked based on Table Engine logic, even grant table engine is disabled by default, if a source is not granted, a prompt of table engine would popup instead, which is misleading. [#65419](https://github.com/ClickHouse/ClickHouse/pull/65419) ([jsc0218](https://github.com/jsc0218)).
|
||||
* Added statistics type `count_min` (count-min sketches) which provide selectivity estimations for equality predicates like `col = 'val'`. Supported data types are string, date, datatime and numeric types. [#65521](https://github.com/ClickHouse/ClickHouse/pull/65521) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Do not pass logs for keeper explicitly in the image to allow overriding. [#65564](https://github.com/ClickHouse/ClickHouse/pull/65564) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Use `Atomic` database by default in `clickhouse-local`. Address items 1 and 5 from [#50647](https://github.com/ClickHouse/ClickHouse/issues/50647). Closes [#44817](https://github.com/ClickHouse/ClickHouse/issues/44817). [#65860](https://github.com/ClickHouse/ClickHouse/pull/65860) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add the `rows_before_aggregation_at_least` statistic to the query response when `rows_before_aggregation` is enabled. This statistic represents the number of rows read before aggregation. In the context of a distributed query, when using the `group by` or `max` aggregation function without a `limit`, `rows_before_aggregation_at_least` can reflect the number of rows hit by the query. [#66084](https://github.com/ClickHouse/ClickHouse/pull/66084) ([morning-color](https://github.com/morning-color)).
|
||||
* Introduced `use_same_password_for_base_backup` settings for `BACKUP` and `RESTORE` queries, allowing to create and restore incremental backups to/from password protected archives. [#66214](https://github.com/ClickHouse/ClickHouse/pull/66214) ([Samuele](https://github.com/sguerrini97)).
|
||||
* Ignore async_load_databases for ATTACH query (previously it was possible for ATTACH to return before the tables had been attached). [#66240](https://github.com/ClickHouse/ClickHouse/pull/66240) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* [Replicated]MergeTreeSink has to properly cancel its delayed_chunk on `onCancel()` method. [#66279](https://github.com/ClickHouse/ClickHouse/pull/66279) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Added logs and metrics for rejected connections (where there are not enough resources). [#66410](https://github.com/ClickHouse/ClickHouse/pull/66410) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Setting `allow_experimental_analyzer` is renamed to `enable_analyzer`. The old name is preserved in a form of an alias. [#66438](https://github.com/ClickHouse/ClickHouse/pull/66438) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Support true UUID type for MongoDB engine. [#66671](https://github.com/ClickHouse/ClickHouse/pull/66671) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Added a new `MergeTree` setting `deduplicate_merge_projection_mode` to control the projections during merges (for specific engines) and `OPTIMIZE DEDUPLICATE` query. Supported options: `throw` (throw an exception in case the projection is not fully supported for *MergeTree engine), `drop` (remove projection during merge if it can't be merged itself consistently) and `rebuild` (rebuild projection from scratch, which is a heavy operation). [#66672](https://github.com/ClickHouse/ClickHouse/pull/66672) ([jsc0218](https://github.com/jsc0218)).
|
||||
* Add replication lag and recovery time metrics. [#66703](https://github.com/ClickHouse/ClickHouse/pull/66703) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Add S3DiskNoKeyErrors metric. [#66704](https://github.com/ClickHouse/ClickHouse/pull/66704) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Ensure COMMENT clause works for all table engines. [#66832](https://github.com/ClickHouse/ClickHouse/pull/66832) ([Joe Lynch](https://github.com/joelynch)).
|
||||
* Update the usage of error code `BAD_ARGUMENTS` and `ILLEGAL_TYPE_OF_ARGUMENT` by more accurate error codes when appropriate. [#66851](https://github.com/ClickHouse/ClickHouse/pull/66851) ([Yohann Jardin](https://github.com/yohannj)).
|
||||
* Function `mapFromArrays` now accepts `Map(K, V)` as first argument, for example: `SELECT mapFromArrays(map('a', 4, 'b', 4), ['aa', 'bb'])` now works and returns `{('a',4):'aa',('b',4):'bb'}`. Also, if the 1st argument is an Array, it can now also be of type `Array(Nullable(T))` or `Array(LowCardinality(Nullable(T)))` as long as the actual array values are not `NULL`. [#67103](https://github.com/ClickHouse/ClickHouse/pull/67103) ([李扬](https://github.com/taiyang-li)).
|
||||
* Read configuration for clickhouse-local from ~/.clickhouse-local. [#67135](https://github.com/ClickHouse/ClickHouse/pull/67135) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Rename setting `input_format_orc_read_use_writer_time_zone` to `input_format_orc_reader_timezone` and allow the user to set the reader timezone. [#67175](https://github.com/ClickHouse/ClickHouse/pull/67175) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Decrease level of 'Socket is not connected' error when HTTP connection immediately reset by peer after connecting, close [#34218](https://github.com/ClickHouse/ClickHouse/issues/34218). [#67177](https://github.com/ClickHouse/ClickHouse/pull/67177) ([vdimir](https://github.com/vdimir)).
|
||||
* Speed up tables removal for `DROP DATABASE` query, increased the default value for `database_catalog_drop_table_concurrency` to 16. [#67228](https://github.com/ClickHouse/ClickHouse/pull/67228) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Add ability to load dashboards for system.dashboards from config (once set, they overrides the default dashboards preset). [#67232](https://github.com/ClickHouse/ClickHouse/pull/67232) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* The window functions in SQL are traditionally in snake case. ClickHouse uses camelCase, so new aliases `denseRank()` and `percentRank()` have been created. These new functions can be called the exact same as the original `dense_rank()` and `percent_rank()` functions. Both snake case and camelCase syntaxes remain usable. A new test for each of the functions has been added as well. This closes [#67042](https://github.com/ClickHouse/ClickHouse/issues/67042) . [#67334](https://github.com/ClickHouse/ClickHouse/pull/67334) ([Peter Nguyen](https://github.com/petern48)).
|
||||
* Autodetect configuration file format if is not .xml, .yml or .yaml. If the file begins with < it might be XML, otherwise it might be YAML. Non regular file just parse as XML such as PIPE: /dev/fd/X. [#67391](https://github.com/ClickHouse/ClickHouse/pull/67391) ([sakulali](https://github.com/sakulali)).
|
||||
* Functions `formatDateTime` and `formatDateTimeInJodaSyntax` now treat their format parameter as optional. If it is not specified, format strings `%Y-%m-%d %H:%i:%s` and `yyyy-MM-dd HH:mm:ss` are assumed. Example: `SELECT parseDateTime('2021-01-04 23:12:34')` now returns DateTime value `2021-01-04 23:12:34` (previously, this threw an exception). [#67399](https://github.com/ClickHouse/ClickHouse/pull/67399) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Automatically retry Keeper requests in KeeperMap if they happen because of timeout or connection loss. [#67448](https://github.com/ClickHouse/ClickHouse/pull/67448) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Rework usage of custom table's disks. [#67684](https://github.com/ClickHouse/ClickHouse/pull/67684) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Various improvements in the advanced dashboard. This closes [#67697](https://github.com/ClickHouse/ClickHouse/issues/67697). This closes [#63407](https://github.com/ClickHouse/ClickHouse/issues/63407). This closes [#51129](https://github.com/ClickHouse/ClickHouse/issues/51129). This closes [#61204](https://github.com/ClickHouse/ClickHouse/issues/61204). [#67701](https://github.com/ClickHouse/ClickHouse/pull/67701) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Avoid allocate too much capacity for array column while writing orc & some minor refactors to make code cleaner. Performance speeds up 15% for array column. [#67879](https://github.com/ClickHouse/ClickHouse/pull/67879) ([李扬](https://github.com/taiyang-li)).
|
||||
* Support OPTIMIZE query on Join table engine to reduce Join tables memory footprint. [#67883](https://github.com/ClickHouse/ClickHouse/pull/67883) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Add replication lag and recovery time metrics. [#67913](https://github.com/ClickHouse/ClickHouse/pull/67913) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Add '-no-pie' to aarch64 Linux builds to allow proper introspection and symbolizing of stacktraces after a ClickHouse restart. [#67916](https://github.com/ClickHouse/ClickHouse/pull/67916) ([filimonov](https://github.com/filimonov)).
|
||||
* Backported in [#68481](https://github.com/ClickHouse/ClickHouse/issues/68481): Added profile events for merges and mutations for better introspection. [#68015](https://github.com/ClickHouse/ClickHouse/pull/68015) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix settings/current_database in system.processes for async BACKUP/RESTORE. [#68163](https://github.com/ClickHouse/ClickHouse/pull/68163) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Remove unnecessary logs for MergeTree that doesn't support replication. [#68238](https://github.com/ClickHouse/ClickHouse/pull/68238) ([Daniil Ivanik](https://github.com/divanik)).
|
||||
* Backported in [#68430](https://github.com/ClickHouse/ClickHouse/issues/68430): Improve schema inference of date times. Now DateTime64 used only when date time has fractional part, otherwise regular DateTime is used. Inference of Date/DateTime is more strict now, especially when `date_time_input_format='best_effort'` to avoid inferring date times from strings in corner cases. [#68382](https://github.com/ClickHouse/ClickHouse/pull/68382) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Fixed `Not-ready Set` in some system tables when filtering using subqueries. [#66018](https://github.com/ClickHouse/ClickHouse/pull/66018) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fixed reading of subcolumns after `ALTER ADD COLUMN` query. [#66243](https://github.com/ClickHouse/ClickHouse/pull/66243) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix boolean literals in query sent to external database (for engines like `PostgreSQL`). [#66282](https://github.com/ClickHouse/ClickHouse/pull/66282) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix formatting of query with aliased JOIN ON expression, e.g. `... JOIN t2 ON (x = y) AS e ORDER BY x` should be formatted as `... JOIN t2 ON ((x = y) AS e) ORDER BY x`. [#66312](https://github.com/ClickHouse/ClickHouse/pull/66312) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix cluster() for inter-server secret (preserve initial user as before). [#66364](https://github.com/ClickHouse/ClickHouse/pull/66364) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix possible runtime error while converting Array field with nulls to Array(Variant). [#66727](https://github.com/ClickHouse/ClickHouse/pull/66727) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix for occasional deadlock in Context::getDDLWorker. [#66843](https://github.com/ClickHouse/ClickHouse/pull/66843) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Fix creating KeeperMap table after an incomplete drop. [#66865](https://github.com/ClickHouse/ClickHouse/pull/66865) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix broken part error while restoring to a `s3_plain_rewritable` disk. [#66881](https://github.com/ClickHouse/ClickHouse/pull/66881) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* In rare cases ClickHouse could consider parts as broken because of some unexpected projections on disk. Now it's fixed. [#66898](https://github.com/ClickHouse/ClickHouse/pull/66898) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix invalid format detection in schema inference that could lead to logical error Format {} doesn't support schema inference. [#66899](https://github.com/ClickHouse/ClickHouse/pull/66899) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix possible deadlock on query cancel with parallel replicas. [#66905](https://github.com/ClickHouse/ClickHouse/pull/66905) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Forbid create as select even when database_replicated_allow_heavy_create is set. It was unconditionally forbidden in 23.12 and accidentally allowed under the setting in unreleased 24.7. [#66980](https://github.com/ClickHouse/ClickHouse/pull/66980) ([vdimir](https://github.com/vdimir)).
|
||||
* Reading from the `numbers` could wrongly throw an exception when the `max_rows_to_read` limit was set. This closes [#66992](https://github.com/ClickHouse/ClickHouse/issues/66992). [#66996](https://github.com/ClickHouse/ClickHouse/pull/66996) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add proper type conversion to lagInFrame and leadInFrame window functions - fixes msan test. [#67091](https://github.com/ClickHouse/ClickHouse/pull/67091) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Only relevant to the experimental Variant data type. Fix crash with Variant + AggregateFunction type. [#67122](https://github.com/ClickHouse/ClickHouse/pull/67122) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* TRUNCATE DATABASE used to stop replication as if it was a DROP DATABASE query, it's fixed. [#67129](https://github.com/ClickHouse/ClickHouse/pull/67129) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Use a separate client context in `clickhouse-local`. [#67133](https://github.com/ClickHouse/ClickHouse/pull/67133) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix error `Cannot convert column because it is non constant in source stream but must be constant in result.` for a query that reads from the `Merge` table over the `Distriburted` table with one shard. [#67146](https://github.com/ClickHouse/ClickHouse/pull/67146) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Correct behavior of `ORDER BY all` with disabled `enable_order_by_all` and parallel replicas (distributed queries as well). [#67153](https://github.com/ClickHouse/ClickHouse/pull/67153) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix wrong usage of input_format_max_bytes_to_read_for_schema_inference in schema cache. [#67157](https://github.com/ClickHouse/ClickHouse/pull/67157) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix the memory leak for count distinct, when exception issued during group by single nullable key. [#67171](https://github.com/ClickHouse/ClickHouse/pull/67171) ([Jet He](https://github.com/compasses)).
|
||||
* This closes [#67156](https://github.com/ClickHouse/ClickHouse/issues/67156). This closes [#66447](https://github.com/ClickHouse/ClickHouse/issues/66447). The bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/62907. [#67178](https://github.com/ClickHouse/ClickHouse/pull/67178) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Fix crash in DistributedAsyncInsert when connection is empty. [#67219](https://github.com/ClickHouse/ClickHouse/pull/67219) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Fix error `Conversion from AggregateFunction(name, Type) to AggregateFunction(name, Nullable(Type)) is not supported`. The bug was caused by the `optimize_rewrite_aggregate_function_with_if` optimization. Fixes [#67112](https://github.com/ClickHouse/ClickHouse/issues/67112). [#67229](https://github.com/ClickHouse/ClickHouse/pull/67229) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix hung query when using empty tuple as lhs of function IN. [#67295](https://github.com/ClickHouse/ClickHouse/pull/67295) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Fix crash of `uniq` and `uniqTheta ` with `tuple()` argument. Closes [#67303](https://github.com/ClickHouse/ClickHouse/issues/67303). [#67306](https://github.com/ClickHouse/ClickHouse/pull/67306) ([flynn](https://github.com/ucasfl)).
|
||||
* It was possible to create a very deep nested JSON data that triggered stack overflow while skipping unknown fields. This closes [#67292](https://github.com/ClickHouse/ClickHouse/issues/67292). [#67324](https://github.com/ClickHouse/ClickHouse/pull/67324) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix attaching ReplicatedMergeTree table after exception during startup. [#67360](https://github.com/ClickHouse/ClickHouse/pull/67360) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix segfault caused by incorrectly detaching from thread group in `Aggregator`. [#67385](https://github.com/ClickHouse/ClickHouse/pull/67385) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix one more case when a non-deterministic function is specified in PK. [#67395](https://github.com/ClickHouse/ClickHouse/pull/67395) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fixed `bloom_filter` index breaking queries with mildly weird conditions like `(k=2)=(k=2)` or `has([1,2,3], k)`. [#67423](https://github.com/ClickHouse/ClickHouse/pull/67423) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Correctly parse file name/URI containing `::` if it's not an archive. [#67433](https://github.com/ClickHouse/ClickHouse/pull/67433) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix wait for tasks in ~WriteBufferFromS3 in case WriteBuffer was cancelled. [#67459](https://github.com/ClickHouse/ClickHouse/pull/67459) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Protect temporary part directories from removing during RESTORE. [#67491](https://github.com/ClickHouse/ClickHouse/pull/67491) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix execution of nested short-circuit functions. [#67520](https://github.com/ClickHouse/ClickHouse/pull/67520) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fixes [#66026](https://github.com/ClickHouse/ClickHouse/issues/66026). Avoid unresolved table function arguments traversal in `ReplaceTableNodeToDummyVisitor`. [#67522](https://github.com/ClickHouse/ClickHouse/pull/67522) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Fix `Logical error: Expected the argument №N of type T to have X rows, but it has 0`. The error could happen in a remote query with constant expression in `GROUP BY` (with a new analyzer). [#67536](https://github.com/ClickHouse/ClickHouse/pull/67536) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix join on tuple with NULLs: Some queries with the new analyzer and `NULL` inside the tuple in the `JOIN ON` section returned incorrect results. [#67538](https://github.com/ClickHouse/ClickHouse/pull/67538) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix redundant reschedule of FileCache::freeSpaceRatioKeepingThreadFunc() in case of full non-evictable cache. [#67540](https://github.com/ClickHouse/ClickHouse/pull/67540) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix inserting into stream like engines (Kafka, RabbitMQ, NATS) through HTTP interface. [#67554](https://github.com/ClickHouse/ClickHouse/pull/67554) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Fix for function `toStartOfWeek` which returned the wrong result with a small `DateTime64` value. [#67558](https://github.com/ClickHouse/ClickHouse/pull/67558) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fix creation of view with recursive CTE. [#67587](https://github.com/ClickHouse/ClickHouse/pull/67587) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Fix `Logical error: 'file_offset_of_buffer_end <= read_until_position'` in filesystem cache. Closes [#57508](https://github.com/ClickHouse/ClickHouse/issues/57508). [#67623](https://github.com/ClickHouse/ClickHouse/pull/67623) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fixes [#62282](https://github.com/ClickHouse/ClickHouse/issues/62282). Removed the call to `convertFieldToString()` and added datatype specific serialization code. Parameterized view substitution was broken for multiple datatypes when parameter value was a function or expression returning datatype instance. [#67654](https://github.com/ClickHouse/ClickHouse/pull/67654) ([Shankar](https://github.com/shiyer7474)).
|
||||
* Fix crash on `percent_rank`. `percent_rank`'s default frame type is changed to `range unbounded preceding and unbounded following`. `IWindowFunction`'s default window frame is considered and now window functions without window frame definition in sql can be put into different `WindowTransfomer`s properly. [#67661](https://github.com/ClickHouse/ClickHouse/pull/67661) ([lgbo](https://github.com/lgbo-ustc)).
|
||||
* Fix reloading SQL UDFs with UNION. Previously, restarting the server could make UDF invalid. [#67665](https://github.com/ClickHouse/ClickHouse/pull/67665) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix possible logical error "Unexpected return type from if" with experimental Variant type and enabled setting `use_variant_as_common_type ` in function if with Tuples and Maps. [#67687](https://github.com/ClickHouse/ClickHouse/pull/67687) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Due to a bug in Linux Kernel, a query can hung in `TimerDescriptor::drain`. This closes [#37686](https://github.com/ClickHouse/ClickHouse/issues/37686). [#67702](https://github.com/ClickHouse/ClickHouse/pull/67702) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix completion of `RESTORE ON CLUSTER` command. [#67720](https://github.com/ClickHouse/ClickHouse/pull/67720) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix dictionary hang in case of CANNOT_SCHEDULE_TASK while loading. [#67751](https://github.com/ClickHouse/ClickHouse/pull/67751) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix potential stack overflow in `JSONMergePatch` function. Renamed this function from `jsonMergePatch` to `JSONMergePatch` because the previous name was wrong. The previous name is still kept for compatibility. Improved diagnostic of errors in the function. This closes [#67304](https://github.com/ClickHouse/ClickHouse/issues/67304). [#67756](https://github.com/ClickHouse/ClickHouse/pull/67756) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Queries like `SELECT count() FROM t WHERE cast(c = 1 or c = 9999 AS Bool) SETTINGS use_skip_indexes=1` with bloom filter indexes on `c` now work correctly. [#67781](https://github.com/ClickHouse/ClickHouse/pull/67781) ([jsc0218](https://github.com/jsc0218)).
|
||||
* Fix wrong aggregation result in some queries with aggregation without keys and filter, close [#67419](https://github.com/ClickHouse/ClickHouse/issues/67419). [#67804](https://github.com/ClickHouse/ClickHouse/pull/67804) ([vdimir](https://github.com/vdimir)).
|
||||
* Validate experimental/suspicious data types in ALTER ADD/MODIFY COLUMN. [#67911](https://github.com/ClickHouse/ClickHouse/pull/67911) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix DateTime64 parsing after constant folding in distributed queries, close [#66773](https://github.com/ClickHouse/ClickHouse/issues/66773). [#67920](https://github.com/ClickHouse/ClickHouse/pull/67920) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix wrong `count()` result when there is non-deterministic function in predicate. [#67922](https://github.com/ClickHouse/ClickHouse/pull/67922) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Fixed the calculation of the maximum thread soft limit in containerized environments where the usable CPU count is limited. [#67963](https://github.com/ClickHouse/ClickHouse/pull/67963) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Now ClickHouse doesn't consider part as broken if projection doesn't exist on disk but exists in `checksums.txt`. [#68003](https://github.com/ClickHouse/ClickHouse/pull/68003) ([alesapin](https://github.com/alesapin)).
|
||||
* Fixed skipping of untouched parts in mutations with new analyzer. Previously with enabled analyzer data in part could be rewritten by mutation even if mutation doesn't affect this part according to predicate. [#68052](https://github.com/ClickHouse/ClickHouse/pull/68052) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fixed a NULL pointer dereference, triggered by a specially crafted query, that crashed the server via hopEnd, hopStart, tumbleEnd, and tumbleStart. [#68098](https://github.com/ClickHouse/ClickHouse/pull/68098) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
|
||||
* Removes an incorrect optimization to remove sorting in subqueries that use `OFFSET`. Fixes [#67906](https://github.com/ClickHouse/ClickHouse/issues/67906). [#68099](https://github.com/ClickHouse/ClickHouse/pull/68099) ([Graham Campbell](https://github.com/GrahamCampbell)).
|
||||
* Attempt to fix `Block structure mismatch in AggregatingStep stream: different types` for aggregate projection optimization. [#68107](https://github.com/ClickHouse/ClickHouse/pull/68107) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#68343](https://github.com/ClickHouse/ClickHouse/issues/68343): Try fix postgres crash when query is cancelled. [#68288](https://github.com/ClickHouse/ClickHouse/pull/68288) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#68400](https://github.com/ClickHouse/ClickHouse/issues/68400): Fix missing sync replica mode in query `SYSTEM SYNC REPLICA`. [#68326](https://github.com/ClickHouse/ClickHouse/pull/68326) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Improved `test_storage_s3` tests: increased `s3_max_single_read_retries` for read from "unstable" s3 source and allowed all tests to run multiple times in a row. [#66896](https://github.com/ClickHouse/ClickHouse/pull/66896) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Integration tests flaky check will not run each test case multiple times to find more issues in tests and make them more reliable. It is using `pytest-repeat` library to run test case multiple times for the same environment. It is important to cleanup tables and other entities in the end of a test case to pass. Repeat works much faster than several pytest runs as it starts necessary containers only once. [#66986](https://github.com/ClickHouse/ClickHouse/pull/66986) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Allow to use CLion with ClickHouse. In previous versions, CLion freezed for a minute on every keypress. This closes [#66994](https://github.com/ClickHouse/ClickHouse/issues/66994). [#66995](https://github.com/ClickHouse/ClickHouse/pull/66995) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Getauxval: avoid crash under sanitizer re-exec due to high aslr entropy. [#67081](https://github.com/ClickHouse/ClickHouse/pull/67081) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Some parts of client code are extracted to a single file and highest possible level optimization is applied to them even for debug builds. This closes: [#65745](https://github.com/ClickHouse/ClickHouse/issues/65745). [#67215](https://github.com/ClickHouse/ClickHouse/pull/67215) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
|
||||
#### NO CL CATEGORY
|
||||
|
||||
* Backported in [#68416](https://github.com/ClickHouse/ClickHouse/issues/68416):. [#68386](https://github.com/ClickHouse/ClickHouse/pull/68386) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
|
||||
#### NO CL ENTRY
|
||||
|
||||
* NO CL ENTRY: 'Revert "Fix for 992 and friends"'. [#66993](https://github.com/ClickHouse/ClickHouse/pull/66993) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* NO CL ENTRY: 'Revert "Revert "Fix for 992 and friends""'. [#67029](https://github.com/ClickHouse/ClickHouse/pull/67029) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* NO CL ENTRY: 'Revert "FuzzQuery table function"'. [#67040](https://github.com/ClickHouse/ClickHouse/pull/67040) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* NO CL ENTRY: 'Revert "Enable `compile_expressions` by default."'. [#67299](https://github.com/ClickHouse/ClickHouse/pull/67299) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* NO CL ENTRY: 'Revert "Slightly better calculation of primary index"'. [#67392](https://github.com/ClickHouse/ClickHouse/pull/67392) ([alesapin](https://github.com/alesapin)).
|
||||
* NO CL ENTRY: 'Revert "Add settings to replace external engines to Null during create"'. [#67507](https://github.com/ClickHouse/ClickHouse/pull/67507) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* NO CL ENTRY: 'Revert "Revert "Add settings to replace external engines to Null during create""'. [#67511](https://github.com/ClickHouse/ClickHouse/pull/67511) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* NO CL ENTRY: 'Revert "Add replication lag and recovery time metrics"'. [#67731](https://github.com/ClickHouse/ClickHouse/pull/67731) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* NO CL ENTRY: 'Revert "Revert "Slightly better calculation of primary index""'. [#67846](https://github.com/ClickHouse/ClickHouse/pull/67846) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* NO CL ENTRY: 'Revert "CI: Strict job timeout 1.5h for tests, 2h for builds"'. [#67986](https://github.com/ClickHouse/ClickHouse/pull/67986) ([Max K.](https://github.com/maxknv)).
|
||||
* NO CL ENTRY: 'Revert "Bump rocksdb from v8.10 to v9.4 + enable jemalloc and liburing"'. [#68014](https://github.com/ClickHouse/ClickHouse/pull/68014) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* NO CL ENTRY: 'Revert "Use `Atomic` database by default in `clickhouse-local`"'. [#68023](https://github.com/ClickHouse/ClickHouse/pull/68023) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* NO CL ENTRY: 'Revert "Refactor tests for (experimental) statistics"'. [#68156](https://github.com/ClickHouse/ClickHouse/pull/68156) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* CI: enable libfuzzer (fixing build and docker). [#61908](https://github.com/ClickHouse/ClickHouse/pull/61908) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Initial implementation of vector similarity index. [#63675](https://github.com/ClickHouse/ClickHouse/pull/63675) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update zlib-ng from 2.0.2 to 2.1.7. [#64489](https://github.com/ClickHouse/ClickHouse/pull/64489) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix 02444_async_broken_outdated_part_loading flakiness. [#64956](https://github.com/ClickHouse/ClickHouse/pull/64956) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* attach_gdb.lib: print more information before all stacks. [#65253](https://github.com/ClickHouse/ClickHouse/pull/65253) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix some perf tests. [#65320](https://github.com/ClickHouse/ClickHouse/pull/65320) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Remove ActionsDAGPtr whenever it is possible. [#65414](https://github.com/ClickHouse/ClickHouse/pull/65414) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Scheduler queue throughput introspection, Fix CPU indication in client. [#65654](https://github.com/ClickHouse/ClickHouse/pull/65654) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Increase timeout in 02122_join_group_by_timeout for tsan build. [#65976](https://github.com/ClickHouse/ClickHouse/pull/65976) ([vdimir](https://github.com/vdimir)).
|
||||
* Remove default values for certificateFile/privateKeyFile/dhParamsFile in keeper config (to avoid annoying errors in logs). [#65978](https://github.com/ClickHouse/ClickHouse/pull/65978) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Update version_date.tsv and changelogs after v24.3.5.46-lts. [#66054](https://github.com/ClickHouse/ClickHouse/pull/66054) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix flaky `02814_currentDatabase_for_table_functions`. [#66111](https://github.com/ClickHouse/ClickHouse/pull/66111) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix possible data-race StorageKafka with statistics_interval_ms>0. [#66311](https://github.com/ClickHouse/ClickHouse/pull/66311) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Avoid unneeded calculation in SeriesPeriodDetect. [#66320](https://github.com/ClickHouse/ClickHouse/pull/66320) ([Ruihang Xia](https://github.com/waynexia)).
|
||||
* It aims to complete [#58630](https://github.com/ClickHouse/ClickHouse/issues/58630). This is made possible by [#60463](https://github.com/ClickHouse/ClickHouse/issues/60463), [#61459](https://github.com/ClickHouse/ClickHouse/issues/61459) and [#60082](https://github.com/ClickHouse/ClickHouse/issues/60082). [#66443](https://github.com/ClickHouse/ClickHouse/pull/66443) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Allow run query instantly in play. [#66457](https://github.com/ClickHouse/ClickHouse/pull/66457) ([Aleksandr Musorin](https://github.com/AVMusorin)).
|
||||
* Bump ICU from v70 to v75. [#66474](https://github.com/ClickHouse/ClickHouse/pull/66474) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Bump RocksDB from v6.29.5 to v7.10.2. [#66475](https://github.com/ClickHouse/ClickHouse/pull/66475) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Bump RocksDB from v7.10.2 to v8.9.1. [#66479](https://github.com/ClickHouse/ClickHouse/pull/66479) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* I believe the error code for this function should not be "NOT_ALLOWED" since it simply is an invalid query and "BAD_QUERY_PARAMETER" is a more reasonable error code for this. [#66491](https://github.com/ClickHouse/ClickHouse/pull/66491) ([Ali](https://github.com/xogoodnow)).
|
||||
* Update gdb to 15.1 (by compiling from sources). [#66494](https://github.com/ClickHouse/ClickHouse/pull/66494) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Ensure that llvm-symbolizer is used for symbolizing sanitizer reports. [#66495](https://github.com/ClickHouse/ClickHouse/pull/66495) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Remove unused local variables. [#66503](https://github.com/ClickHouse/ClickHouse/pull/66503) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* This will solve a lot of problems with inconsistent formatting. And it opens the path for [#65753](https://github.com/ClickHouse/ClickHouse/issues/65753). This closes [#66807](https://github.com/ClickHouse/ClickHouse/issues/66807). This closes [#61611](https://github.com/ClickHouse/ClickHouse/issues/61611). This closes [#61711](https://github.com/ClickHouse/ClickHouse/issues/61711). This closes [#67445](https://github.com/ClickHouse/ClickHouse/issues/67445). [#66506](https://github.com/ClickHouse/ClickHouse/pull/66506) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Rename Context::getSettings() to Context::getSettingsCopy(). [#66528](https://github.com/ClickHouse/ClickHouse/pull/66528) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Uninteresting change: introducing `ClientApplicationBase`. [#66549](https://github.com/ClickHouse/ClickHouse/pull/66549) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Respond to parallel replicas protocol requests with priority on initiator. [#66618](https://github.com/ClickHouse/ClickHouse/pull/66618) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix bad code: it was catching exceptions. [#66628](https://github.com/ClickHouse/ClickHouse/pull/66628) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Dump all memory stats in CgroupsMemoryUsageObserver on hitting the limit. [#66732](https://github.com/ClickHouse/ClickHouse/pull/66732) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Save writer thread id in shared mutex for debugging. [#66745](https://github.com/ClickHouse/ClickHouse/pull/66745) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Increase asio pool size in case the server is tiny. [#66761](https://github.com/ClickHouse/ClickHouse/pull/66761) ([alesapin](https://github.com/alesapin)).
|
||||
* Looks like it runs too many mutations sometimes and fails to process them within the timeout. So if a query waits for mutations - the test fails. [#66785](https://github.com/ClickHouse/ClickHouse/pull/66785) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Better diagnostics in functional tests. [#66790](https://github.com/ClickHouse/ClickHouse/pull/66790) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bad test `01042_system_reload_dictionary_reloads_completely`. [#66811](https://github.com/ClickHouse/ClickHouse/pull/66811) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Something is strange with the test about refreshable materialized views. [#66816](https://github.com/ClickHouse/ClickHouse/pull/66816) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Randomize `trace_profile_events`. [#66821](https://github.com/ClickHouse/ClickHouse/pull/66821) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Stop ignoring SIGSEGV in GDB. [#66847](https://github.com/ClickHouse/ClickHouse/pull/66847) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Followup [#66725](https://github.com/ClickHouse/ClickHouse/issues/66725). [#66869](https://github.com/ClickHouse/ClickHouse/pull/66869) ([vdimir](https://github.com/vdimir)).
|
||||
* When executing queries with parallel replicas that involve only a subset of nodes within a shard, the current behavior is that if all participating replicas are unavailable, the query completes without any errors but returns no results. Referencing issue [#65467](https://github.com/ClickHouse/ClickHouse/issues/65467), this pull request addresses the issue where only a portion of the nodes in a shard are participating in the execution. [#66880](https://github.com/ClickHouse/ClickHouse/pull/66880) ([zoomxi](https://github.com/zoomxi)).
|
||||
* Speed up stateful tests setup. [#66886](https://github.com/ClickHouse/ClickHouse/pull/66886) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Functions [h-r]*: Iterate over input_rows_count where appropriate. [#66897](https://github.com/ClickHouse/ClickHouse/pull/66897) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Stateless tests: Change status of failed tests in case of server crash and add no-parallel to high-load tests. [#66901](https://github.com/ClickHouse/ClickHouse/pull/66901) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Fix performance test about the generateRandom table function, supposedly. [#66906](https://github.com/ClickHouse/ClickHouse/pull/66906) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bad tests `share_big_sets`, CC @davenger. [#66908](https://github.com/ClickHouse/ClickHouse/pull/66908) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Speed up mutations for non-replicated MergeTree a bit. [#66909](https://github.com/ClickHouse/ClickHouse/pull/66909) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Speed up mutations for non-replicated MergeTree significantly. [#66911](https://github.com/ClickHouse/ClickHouse/pull/66911) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix views over distributed tables with Analyzer. [#66912](https://github.com/ClickHouse/ClickHouse/pull/66912) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* [CI fest] Try to fix `test_broken_projections/test.py::test_broken_ignored_replicated`. [#66915](https://github.com/ClickHouse/ClickHouse/pull/66915) ([Andrey Zvonov](https://github.com/zvonand)).
|
||||
* Decrease rate limit in `01923_network_receive_time_metric_insert`. [#66924](https://github.com/ClickHouse/ClickHouse/pull/66924) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Grouparrayintersect: fix serialization bug. [#66928](https://github.com/ClickHouse/ClickHouse/pull/66928) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Update version after release branch. [#66929](https://github.com/ClickHouse/ClickHouse/pull/66929) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Un-flake test_runtime_configurable_cache_size. [#66934](https://github.com/ClickHouse/ClickHouse/pull/66934) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* fix unit tests ResolvePoolTest with timeouts. [#66953](https://github.com/ClickHouse/ClickHouse/pull/66953) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Split slow test 03036_dynamic_read_subcolumns. [#66954](https://github.com/ClickHouse/ClickHouse/pull/66954) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* CI: Fixes docker server build for release branches. [#66955](https://github.com/ClickHouse/ClickHouse/pull/66955) ([Max K.](https://github.com/maxknv)).
|
||||
* Addressing issue [#64936](https://github.com/ClickHouse/ClickHouse/issues/64936). [#66973](https://github.com/ClickHouse/ClickHouse/pull/66973) ([alesapin](https://github.com/alesapin)).
|
||||
* Add initial 24.7 changelog. [#66976](https://github.com/ClickHouse/ClickHouse/pull/66976) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Apply libunwind fix. [#66977](https://github.com/ClickHouse/ClickHouse/pull/66977) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* CI: Add logs for debugging. [#66979](https://github.com/ClickHouse/ClickHouse/pull/66979) ([Max K.](https://github.com/maxknv)).
|
||||
* [CI Fest] Split dynamic tests and rewrite them from sh to sql to avoid timeouts. [#66981](https://github.com/ClickHouse/ClickHouse/pull/66981) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Split 01508_partition_pruning_long. [#66983](https://github.com/ClickHouse/ClickHouse/pull/66983) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* [CI Fest] Fix use-of-uninitialized-value in JSONExtract* numeric functions. [#66984](https://github.com/ClickHouse/ClickHouse/pull/66984) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* It should fix SQLancer checks, but for some reason we stopped invalidating cache for docker builds and fix was not published to our CI for a while. [#66987](https://github.com/ClickHouse/ClickHouse/pull/66987) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Fixes [#66941](https://github.com/ClickHouse/ClickHouse/issues/66941). [#66991](https://github.com/ClickHouse/ClickHouse/pull/66991) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Remove the support for Kerberized HDFS. [#66998](https://github.com/ClickHouse/ClickHouse/pull/66998) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* CI: Fix for workflow results parsing. [#67000](https://github.com/ClickHouse/ClickHouse/pull/67000) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix flaky `01454_storagememory_data_race_challenge`. [#67003](https://github.com/ClickHouse/ClickHouse/pull/67003) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* CI: Jepsen Workflow fix for skipped builds and observability. [#67004](https://github.com/ClickHouse/ClickHouse/pull/67004) ([Max K.](https://github.com/maxknv)).
|
||||
* bugfix AttachedTable counting not symmetry, and adding some test logs…. [#67007](https://github.com/ClickHouse/ClickHouse/pull/67007) ([Xu Jia](https://github.com/XuJia0210)).
|
||||
* CI: Automerge when required and non-required checks completed. [#67008](https://github.com/ClickHouse/ClickHouse/pull/67008) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix test `very_long_arrays`. [#67009](https://github.com/ClickHouse/ClickHouse/pull/67009) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Try to fix exception logging in destructors of static objects. [#67016](https://github.com/ClickHouse/ClickHouse/pull/67016) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* [Green CI] Fix test test_storage_azure_blob_storage. [#67019](https://github.com/ClickHouse/ClickHouse/pull/67019) ([Daniil Ivanik](https://github.com/divanik)).
|
||||
* Integration tests: fix flaky tests `test_backup_restore_on_cluster/test_concurrency.py` & `test_manipulate_statistics/test.py`. [#67027](https://github.com/ClickHouse/ClickHouse/pull/67027) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* [Green CI] Fix test test_storage_s3_queue/test.py::test_max_set_age. [#67035](https://github.com/ClickHouse/ClickHouse/pull/67035) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Test for alter select with parallel replicas. [#67041](https://github.com/ClickHouse/ClickHouse/pull/67041) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Split query into multiple queries to consume less memory at once + use less data. Fixes [#67034](https://github.com/ClickHouse/ClickHouse/issues/67034). [#67044](https://github.com/ClickHouse/ClickHouse/pull/67044) ([alesapin](https://github.com/alesapin)).
|
||||
* Disable setting `optimize_functions_to_subcolumns`. [#67046](https://github.com/ClickHouse/ClickHouse/pull/67046) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Increase max allocation size for sanitizers. [#67049](https://github.com/ClickHouse/ClickHouse/pull/67049) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* rewrite 01171 test. [#67054](https://github.com/ClickHouse/ClickHouse/pull/67054) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Add `**` to `hdfs` docs, add test for `**` in `hdfs`. [#67064](https://github.com/ClickHouse/ClickHouse/pull/67064) ([Andrey Zvonov](https://github.com/zvonand)).
|
||||
* Very sad failure: ``` 2024.07.24 13:28:45.517777 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> executeQuery: (from 172.16.11.1:55890) OPTIMIZE TABLE replicated_mt FINAL (stage: Complete) 2024.07.24 13:28:45.525945 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Trace> default.replicated_mt (ReplicatedMergeTreeQueue): Waiting for 4 entries to be processed: queue-0000000004, queue-0000000002, queue-0000000001, queue-0000000000 2024.07.24 13:29:15.528024 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e) (MergerMutator): Selected 3 parts from all_0_0_0 to all_2_2_0 2024.07.24 13:29:15.530736 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Trace> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Created log entry /clickhouse/tables/replicated_mt/log/log-0000000004 for merge all_0_2_1 2024.07.24 13:29:15.530873 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Waiting for node1 to process log entry 2024.07.24 13:29:15.530919 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Waiting for node1 to pull log-0000000004 to queue 2024.07.24 13:29:15.534286 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Looking for node corresponding to log-0000000004 in node1 queue 2024.07.24 13:29:15.534793 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Waiting for queue-0000000005 to disappear from node1 queue 2024.07.24 13:29:15.585533 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> TCPHandler: Processed in 30.067804125 sec. ```. [#67067](https://github.com/ClickHouse/ClickHouse/pull/67067) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix flaky `test_seekable_formats_url` and `test_seekable_formats` S3 storage tests. [#67070](https://github.com/ClickHouse/ClickHouse/pull/67070) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* CI: Docker server build fix for new release workflow. [#67075](https://github.com/ClickHouse/ClickHouse/pull/67075) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix 2680 flasky. [#67078](https://github.com/ClickHouse/ClickHouse/pull/67078) ([jsc0218](https://github.com/jsc0218)).
|
||||
* [CI Fest] Fix flaky 02447_drop_replica test. [#67085](https://github.com/ClickHouse/ClickHouse/pull/67085) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fixes [#67030](https://github.com/ClickHouse/ClickHouse/issues/67030). [#67086](https://github.com/ClickHouse/ClickHouse/pull/67086) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Increase timeout for curl in tests. [#67089](https://github.com/ClickHouse/ClickHouse/pull/67089) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Try calculating available memory if ClickHouse is bound to subset of NUMA nodes. [#67098](https://github.com/ClickHouse/ClickHouse/pull/67098) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* A more precise way of tracking flushing time in 01246_buffer_flush. [#67099](https://github.com/ClickHouse/ClickHouse/pull/67099) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Do not fail CheckReadyForMerge on failed Tests_2 (non-required jobs) Do not skip CiBuddy report step on failures. [#67101](https://github.com/ClickHouse/ClickHouse/pull/67101) ([Max K.](https://github.com/maxknv)).
|
||||
* Tststs_1 - for all required checks Tststs_2 - for all non-required checks (normal mode) Tststs_2_ww - for all non-required checks (woolenwolfdog mode). [#67104](https://github.com/ClickHouse/ClickHouse/pull/67104) ([Max K.](https://github.com/maxknv)).
|
||||
* Functions [s-t]*: Iterate over input_rows_count where appropriate. [#67105](https://github.com/ClickHouse/ClickHouse/pull/67105) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Reintroduce 02805_distributed_queries_timeouts. [#67106](https://github.com/ClickHouse/ClickHouse/pull/67106) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Added some tests in relation with [#54881](https://github.com/ClickHouse/ClickHouse/issues/54881). [#67110](https://github.com/ClickHouse/ClickHouse/pull/67110) ([max-vostrikov](https://github.com/max-vostrikov)).
|
||||
* Reintroduce 03002_part_log_rmt_fetch_*_error tests without flakiness and less time. [#67113](https://github.com/ClickHouse/ClickHouse/pull/67113) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Improve tag matching in backport scripts. [#67118](https://github.com/ClickHouse/ClickHouse/pull/67118) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fixes [#67111](https://github.com/ClickHouse/ClickHouse/issues/67111). [#67121](https://github.com/ClickHouse/ClickHouse/pull/67121) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Increase lock_acquire_timeout_for_background_operations setting in dynamic merges tests. [#67126](https://github.com/ClickHouse/ClickHouse/pull/67126) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Attempt to fix flakiness of some window view tests. [#67130](https://github.com/ClickHouse/ClickHouse/pull/67130) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update assert in cache. [#67138](https://github.com/ClickHouse/ClickHouse/pull/67138) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix test `00673_subquery_prepared_set_performance`. [#67141](https://github.com/ClickHouse/ClickHouse/pull/67141) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fixes [#67047](https://github.com/ClickHouse/ClickHouse/issues/67047). [#67142](https://github.com/ClickHouse/ClickHouse/pull/67142) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Lower max allocation size in query fuzzer. [#67145](https://github.com/ClickHouse/ClickHouse/pull/67145) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fixes [#66966](https://github.com/ClickHouse/ClickHouse/issues/66966). [#67147](https://github.com/ClickHouse/ClickHouse/pull/67147) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Try fix `02481_async_insert_race_long` flakiness. [#67148](https://github.com/ClickHouse/ClickHouse/pull/67148) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Rename (unreleased) bad setting. [#67149](https://github.com/ClickHouse/ClickHouse/pull/67149) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Uncomment accidentally commented out code in QueryProfiler. [#67152](https://github.com/ClickHouse/ClickHouse/pull/67152) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Try to fix 2572. [#67158](https://github.com/ClickHouse/ClickHouse/pull/67158) ([jsc0218](https://github.com/jsc0218)).
|
||||
* Fix benign data race in ZooKeeper. [#67164](https://github.com/ClickHouse/ClickHouse/pull/67164) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove too long unit test. [#67168](https://github.com/ClickHouse/ClickHouse/pull/67168) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix `00705_drop_create_merge_tree`. [#67170](https://github.com/ClickHouse/ClickHouse/pull/67170) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix stacktrace cache. [#67173](https://github.com/ClickHouse/ClickHouse/pull/67173) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fixes [#67151](https://github.com/ClickHouse/ClickHouse/issues/67151). [#67174](https://github.com/ClickHouse/ClickHouse/pull/67174) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Make 02908_many_requests_to_system_replicas less stressful. [#67176](https://github.com/ClickHouse/ClickHouse/pull/67176) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Reduce max time of 00763_long_lock_buffer_alter_destination_table. [#67185](https://github.com/ClickHouse/ClickHouse/pull/67185) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Do not count AttachedTable for tables in information schema databases. [#67187](https://github.com/ClickHouse/ClickHouse/pull/67187) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Verbose output for 03203_client_benchmark_options. [#67188](https://github.com/ClickHouse/ClickHouse/pull/67188) ([vdimir](https://github.com/vdimir)).
|
||||
* Split test 02967_parallel_replicas_join_algo_and_analyzer. [#67211](https://github.com/ClickHouse/ClickHouse/pull/67211) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix flaky `test_pkill_query_log` (tsan). [#67223](https://github.com/ClickHouse/ClickHouse/pull/67223) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Remove integration test `test_broken_projections_in_backups_1`. [#67231](https://github.com/ClickHouse/ClickHouse/pull/67231) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Debug logging for [#67002](https://github.com/ClickHouse/ClickHouse/issues/67002). [#67233](https://github.com/ClickHouse/ClickHouse/pull/67233) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix oss-fuzz build. [#67235](https://github.com/ClickHouse/ClickHouse/pull/67235) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix flaky 00180_no_seek_avoiding_when_reading_from_cache. [#67236](https://github.com/ClickHouse/ClickHouse/pull/67236) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* English. [#67258](https://github.com/ClickHouse/ClickHouse/pull/67258) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove strange code. [#67260](https://github.com/ClickHouse/ClickHouse/pull/67260) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix MSan report in DatabaseReplicated. [#67262](https://github.com/ClickHouse/ClickHouse/pull/67262) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `02310_clickhouse_local_INSERT_progress_profile_events`. [#67264](https://github.com/ClickHouse/ClickHouse/pull/67264) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove test `02982_aggregation_states_destruction`. [#67266](https://github.com/ClickHouse/ClickHouse/pull/67266) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix OOM in test runs. [#67268](https://github.com/ClickHouse/ClickHouse/pull/67268) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove harmful stuff from tests. [#67275](https://github.com/ClickHouse/ClickHouse/pull/67275) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `03201_variant_null_map_subcolumn`. [#67276](https://github.com/ClickHouse/ClickHouse/pull/67276) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Split `01651_lc_insert_tiny_log`. [#67279](https://github.com/ClickHouse/ClickHouse/pull/67279) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Debug test `02490_benchmark_max_consecutive_errors`. [#67281](https://github.com/ClickHouse/ClickHouse/pull/67281) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bad test `02833_concurrrent_sessions`. [#67282](https://github.com/ClickHouse/ClickHouse/pull/67282) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add a separate test for exception handling. [#67283](https://github.com/ClickHouse/ClickHouse/pull/67283) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Debug test `01600_parts_states_metrics_long`. [#67284](https://github.com/ClickHouse/ClickHouse/pull/67284) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Faster test `02231_buffer_aggregate_states_leak`. [#67285](https://github.com/ClickHouse/ClickHouse/pull/67285) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix curiosities in `TimerDescriptor`. [#67287](https://github.com/ClickHouse/ClickHouse/pull/67287) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add retries to test `02911_backup_restore_keeper_map`. [#67290](https://github.com/ClickHouse/ClickHouse/pull/67290) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Functions: Iterate over input_rows_count where appropriate. [#67294](https://github.com/ClickHouse/ClickHouse/pull/67294) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add documentation for `compile_expressions`. [#67300](https://github.com/ClickHouse/ClickHouse/pull/67300) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Wrap log lines in the CI report for functional tests. [#67301](https://github.com/ClickHouse/ClickHouse/pull/67301) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bad test `02050_client_profile_events`. [#67309](https://github.com/ClickHouse/ClickHouse/pull/67309) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* I do not want to think about this code. [#67312](https://github.com/ClickHouse/ClickHouse/pull/67312) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `00940_max_parts_in_total`. [#67313](https://github.com/ClickHouse/ClickHouse/pull/67313) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Tests for Kafka cannot run in parallel. [#67315](https://github.com/ClickHouse/ClickHouse/pull/67315) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add a test for [#66285](https://github.com/ClickHouse/ClickHouse/issues/66285). [#67325](https://github.com/ClickHouse/ClickHouse/pull/67325) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Follow-up for [#67301](https://github.com/ClickHouse/ClickHouse/issues/67301). [#67327](https://github.com/ClickHouse/ClickHouse/pull/67327) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add a test for [#61659](https://github.com/ClickHouse/ClickHouse/issues/61659). [#67332](https://github.com/ClickHouse/ClickHouse/pull/67332) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix integration test `test_backup_restore_on_cluster/test_disallow_concurrency`. [#67336](https://github.com/ClickHouse/ClickHouse/pull/67336) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Faster and less flaky 01246_buffer_flush (by using HTTP over clickhouse-client). [#67340](https://github.com/ClickHouse/ClickHouse/pull/67340) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix: data race in TCPHandler on socket timeouts settings. [#67341](https://github.com/ClickHouse/ClickHouse/pull/67341) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* CI: Print stdout, stderr for docker pull command. [#67343](https://github.com/ClickHouse/ClickHouse/pull/67343) ([Max K.](https://github.com/maxknv)).
|
||||
* Followup [#67290](https://github.com/ClickHouse/ClickHouse/issues/67290). [#67348](https://github.com/ClickHouse/ClickHouse/pull/67348) ([vdimir](https://github.com/vdimir)).
|
||||
* Skip parallel for `test_storage_kerberized_kafka`. [#67349](https://github.com/ClickHouse/ClickHouse/pull/67349) ([Andrey Zvonov](https://github.com/zvonand)).
|
||||
* Don't use PeekableReadBuffer in JSONAsObject format. [#67354](https://github.com/ClickHouse/ClickHouse/pull/67354) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* This closes: [#57316](https://github.com/ClickHouse/ClickHouse/issues/57316). [#67355](https://github.com/ClickHouse/ClickHouse/pull/67355) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Remove duplicated tests. [#67357](https://github.com/ClickHouse/ClickHouse/pull/67357) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Release branch was not detected properly and job which must run on release branch could be reused from feature branches. PR Fixes detection of release branches. [#67358](https://github.com/ClickHouse/ClickHouse/pull/67358) ([Max K.](https://github.com/maxknv)).
|
||||
* Disable some Dynamic tests under sanitizers, rewrite 03202_dynamic_null_map_subcolumn to sql. [#67359](https://github.com/ClickHouse/ClickHouse/pull/67359) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Add no-distributed-cache tag in tests. [#67361](https://github.com/ClickHouse/ClickHouse/pull/67361) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Trying to fix test_cache_evicted_by_temporary_data and print debug info. [#67362](https://github.com/ClickHouse/ClickHouse/pull/67362) ([vdimir](https://github.com/vdimir)).
|
||||
* Try to fix: ALL_CONNECTION_TRIES_FAILED with parallel replicas. [#67389](https://github.com/ClickHouse/ClickHouse/pull/67389) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix bad test `01036_no_superfluous_dict_reload_on_create_database`. [#67390](https://github.com/ClickHouse/ClickHouse/pull/67390) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Adding `SelectedPartsTotal` and `SelectedMarksTotal` as new ProfileEvents. [#67393](https://github.com/ClickHouse/ClickHouse/pull/67393) ([Jordi Villar](https://github.com/jrdi)).
|
||||
* Print debug info in `test_storage_s3_queue/test.py::test_shards_distributed`. [#67394](https://github.com/ClickHouse/ClickHouse/pull/67394) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Mute degraded perf test. [#67396](https://github.com/ClickHouse/ClickHouse/pull/67396) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Debug TimerDescriptor. [#67397](https://github.com/ClickHouse/ClickHouse/pull/67397) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove has_single_port property from plan stream. [#67398](https://github.com/ClickHouse/ClickHouse/pull/67398) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix typo. [#67400](https://github.com/ClickHouse/ClickHouse/pull/67400) ([Halersson Paris](https://github.com/halersson)).
|
||||
* CI: Fix build results for release branches. [#67402](https://github.com/ClickHouse/ClickHouse/pull/67402) ([Max K.](https://github.com/maxknv)).
|
||||
* Disable 02932_refreshable_materialized_views. [#67404](https://github.com/ClickHouse/ClickHouse/pull/67404) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Follow-up to [#67294](https://github.com/ClickHouse/ClickHouse/issues/67294). [#67405](https://github.com/ClickHouse/ClickHouse/pull/67405) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix DWARF range list parsing in stack symbolizer. [#67417](https://github.com/ClickHouse/ClickHouse/pull/67417) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Make Dwarf::findAddress() fallback slow path less slow. [#67418](https://github.com/ClickHouse/ClickHouse/pull/67418) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix inconsistent formatting of CODEC and STATISTICS. [#67421](https://github.com/ClickHouse/ClickHouse/pull/67421) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Reduced complexity of the test 02832_alter_max_sessions_for_user. [#67425](https://github.com/ClickHouse/ClickHouse/pull/67425) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
|
||||
* Remove obsolete `--multiquery` parameter from tests. [#67435](https://github.com/ClickHouse/ClickHouse/pull/67435) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix public backports. [#67439](https://github.com/ClickHouse/ClickHouse/pull/67439) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Bump Azure from v1.12 to v1.13. [#67446](https://github.com/ClickHouse/ClickHouse/pull/67446) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* 24.7 add missing documentation and testing. [#67454](https://github.com/ClickHouse/ClickHouse/pull/67454) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Use correct order of fields in `StorageURLSource`. [#67455](https://github.com/ClickHouse/ClickHouse/pull/67455) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* run 01171 test in parallel. [#67470](https://github.com/ClickHouse/ClickHouse/pull/67470) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* [Green CI] Fix WriteBuffer destructor when finalize has failed for MergeTreeDeduplicationLog::shutdown. [#67474](https://github.com/ClickHouse/ClickHouse/pull/67474) ([Alexey Katsman](https://github.com/alexkats)).
|
||||
* Reduce 02473_multistep_prewhere run time. [#67475](https://github.com/ClickHouse/ClickHouse/pull/67475) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Update version_date.tsv and changelogs after v24.7.1.2915-stable. [#67483](https://github.com/ClickHouse/ClickHouse/pull/67483) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Revert [#61750](https://github.com/ClickHouse/ClickHouse/issues/61750) "Improve JSONEachRow reading by ignoring the keys case". [#67484](https://github.com/ClickHouse/ClickHouse/pull/67484) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Disable parallel run for `01923_network_receive_time_metric_insert.sh`. [#67492](https://github.com/ClickHouse/ClickHouse/pull/67492) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Fix test `test_backup_restore_on_cluster/test.py::test_mutation`. [#67494](https://github.com/ClickHouse/ClickHouse/pull/67494) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* [Green CI] Fix potentially flaky test_mask_sensitive_info integration test. [#67506](https://github.com/ClickHouse/ClickHouse/pull/67506) ([Alexey Katsman](https://github.com/alexkats)).
|
||||
* [Green CI] Test `test_storage_azure_blob_storage/test.py` is flaky. [#67512](https://github.com/ClickHouse/ClickHouse/pull/67512) ([Daniil Ivanik](https://github.com/divanik)).
|
||||
* Prepare Release workflow for production. [#67523](https://github.com/ClickHouse/ClickHouse/pull/67523) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix upgrade check. [#67524](https://github.com/ClickHouse/ClickHouse/pull/67524) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* [Green CI] test 03164_s3_settings_for_queries_and_merges is flaky. [#67535](https://github.com/ClickHouse/ClickHouse/pull/67535) ([Daniil Ivanik](https://github.com/divanik)).
|
||||
* Log message and increased concurrency for table removal. [#67537](https://github.com/ClickHouse/ClickHouse/pull/67537) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix AsyncLoader destruction race. [#67553](https://github.com/ClickHouse/ClickHouse/pull/67553) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Add an assert into TimerDescriptor. [#67555](https://github.com/ClickHouse/ClickHouse/pull/67555) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Re-enable ICU on s390/x. [#67557](https://github.com/ClickHouse/ClickHouse/pull/67557) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update version_date.tsv and changelogs after v24.4.4.107-stable. [#67559](https://github.com/ClickHouse/ClickHouse/pull/67559) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Sometimes we fail with timeout in stateless tests and the reason for that seems to be in `stop_logs_replication` step. Add a check for timeout here. [#67560](https://github.com/ClickHouse/ClickHouse/pull/67560) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Miscellaneous. [#67564](https://github.com/ClickHouse/ClickHouse/pull/67564) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* enable parallel_view_processing in perf tests. [#67565](https://github.com/ClickHouse/ClickHouse/pull/67565) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Fix flaky `test_system_kafka_consumers_rebalance`. [#67566](https://github.com/ClickHouse/ClickHouse/pull/67566) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Update version_date.tsv and changelogs after v24.7.2.13-stable. [#67586](https://github.com/ClickHouse/ClickHouse/pull/67586) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix 01811_storage_buffer_flush_parameters flakiness. [#67589](https://github.com/ClickHouse/ClickHouse/pull/67589) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix test_zookeeper_config_load_balancing after adding the xdist worker name to the instance. [#67590](https://github.com/ClickHouse/ClickHouse/pull/67590) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Update minio in integration tests. [#67595](https://github.com/ClickHouse/ClickHouse/pull/67595) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* added tests for page index in parquet files. [#67596](https://github.com/ClickHouse/ClickHouse/pull/67596) ([max-vostrikov](https://github.com/max-vostrikov)).
|
||||
* Update check_rabbitmq_is_available. [#67597](https://github.com/ClickHouse/ClickHouse/pull/67597) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix 02434_cancel_insert_when_client_dies. [#67600](https://github.com/ClickHouse/ClickHouse/pull/67600) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix 02910_bad_logs_level_in_local in fast tests. [#67603](https://github.com/ClickHouse/ClickHouse/pull/67603) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix 01605_adaptive_granularity_block_borders. [#67605](https://github.com/ClickHouse/ClickHouse/pull/67605) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Update CHANGELOG.md. [#67607](https://github.com/ClickHouse/ClickHouse/pull/67607) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove some `no-parallel` tags from tests. [#67610](https://github.com/ClickHouse/ClickHouse/pull/67610) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Update README.md. [#67613](https://github.com/ClickHouse/ClickHouse/pull/67613) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* Try fix 03143_asof_join_ddb_long. [#67620](https://github.com/ClickHouse/ClickHouse/pull/67620) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Don't run ASAN unit tests under gdb. [#67622](https://github.com/ClickHouse/ClickHouse/pull/67622) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix crash in KeyCondition::cloneASTWithInversionPushDown() caused by type change. [#67641](https://github.com/ClickHouse/ClickHouse/pull/67641) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix race condition between ProcessList and Settings. [#67645](https://github.com/ClickHouse/ClickHouse/pull/67645) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* `02481_async_insert_race_long.sh` flakiness fixes. [#67650](https://github.com/ClickHouse/ClickHouse/pull/67650) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Fixes [#67651](https://github.com/ClickHouse/ClickHouse/issues/67651). [#67653](https://github.com/ClickHouse/ClickHouse/pull/67653) ([pufit](https://github.com/pufit)).
|
||||
* Fix flaky `test_replicated_table_attach`. [#67658](https://github.com/ClickHouse/ClickHouse/pull/67658) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Update version_date.tsv and changelogs after v24.4.4.113-stable. [#67659](https://github.com/ClickHouse/ClickHouse/pull/67659) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Even better healthcheck for ldap. [#67667](https://github.com/ClickHouse/ClickHouse/pull/67667) ([Andrey Zvonov](https://github.com/zvonand)).
|
||||
* Fix 03203_client_benchmark_options. [#67671](https://github.com/ClickHouse/ClickHouse/pull/67671) ([vdimir](https://github.com/vdimir)).
|
||||
* Integration tests: fix ports clashing problem. [#67672](https://github.com/ClickHouse/ClickHouse/pull/67672) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Remove some `no-parallel` tags from tests (Part 2). [#67673](https://github.com/ClickHouse/ClickHouse/pull/67673) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Use FunctionArgumentDescriptors for bitSlice. [#67674](https://github.com/ClickHouse/ClickHouse/pull/67674) ([Lennard Eijsackers](https://github.com/Blokje5)).
|
||||
* Update version_date.tsv and changelog after v24.3.6.48-lts. [#67677](https://github.com/ClickHouse/ClickHouse/pull/67677) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Avoid ddl queue timeout in 02313_filesystem_cache_seeks. [#67680](https://github.com/ClickHouse/ClickHouse/pull/67680) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix bad log message in sort description. [#67690](https://github.com/ClickHouse/ClickHouse/pull/67690) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Update version_date.tsv and changelog after v23.8.16.40-lts. [#67692](https://github.com/ClickHouse/ClickHouse/pull/67692) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix check names in test reports and the CI Logs database. [#67696](https://github.com/ClickHouse/ClickHouse/pull/67696) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* CI: Minor refactoring in ci_utils. [#67706](https://github.com/ClickHouse/ClickHouse/pull/67706) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix 01042_system_reload_dictionary_reloads_completely flakiness. [#67719](https://github.com/ClickHouse/ClickHouse/pull/67719) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix test `00002_log_and_exception_messages_formatting`. [#67723](https://github.com/ClickHouse/ClickHouse/pull/67723) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `02789_reading_from_s3_with_connection_pool`. [#67726](https://github.com/ClickHouse/ClickHouse/pull/67726) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix strange code in HostResolvePool. [#67727](https://github.com/ClickHouse/ClickHouse/pull/67727) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix a typo. [#67729](https://github.com/ClickHouse/ClickHouse/pull/67729) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Smart handling of processes leftovers in tests. [#67737](https://github.com/ClickHouse/ClickHouse/pull/67737) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix test retries. [#67738](https://github.com/ClickHouse/ClickHouse/pull/67738) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fill only selected columns from system.clusters. [#67739](https://github.com/ClickHouse/ClickHouse/pull/67739) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Bump NuRaft (to properly catch thread exceptions). [#67740](https://github.com/ClickHouse/ClickHouse/pull/67740) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Try to fix RabbitMQ test failures. [#67743](https://github.com/ClickHouse/ClickHouse/pull/67743) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Stateless tests: attempt to fix timeouts of `02473_multistep_prewhere* 00411_long_accurate_number_comparison*`. [#67746](https://github.com/ClickHouse/ClickHouse/pull/67746) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Fix test_ttl_move::test_alter_with_merge_work flakiness. [#67747](https://github.com/ClickHouse/ClickHouse/pull/67747) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* ci: better stateless runner (correctly collect artifacts and also some basic errors capturing). [#67752](https://github.com/ClickHouse/ClickHouse/pull/67752) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Introduce `no-flaky-check` tag. [#67755](https://github.com/ClickHouse/ClickHouse/pull/67755) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Whitespaces. [#67771](https://github.com/ClickHouse/ClickHouse/pull/67771) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* [RFC] Print original query for AST formatting check on CI. [#67776](https://github.com/ClickHouse/ClickHouse/pull/67776) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix test `02833_concurrent_sessions`, Fix test `02835_drop_user_during_session`. [#67779](https://github.com/ClickHouse/ClickHouse/pull/67779) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix diagnostics in the test script. [#67780](https://github.com/ClickHouse/ClickHouse/pull/67780) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `02231_bloom_filter_sizing`. [#67784](https://github.com/ClickHouse/ClickHouse/pull/67784) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fixed session_log related tests race condition on logout. [#67785](https://github.com/ClickHouse/ClickHouse/pull/67785) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
|
||||
* ci/stateless: fix artifacts post-processing and detect if something failed there. [#67791](https://github.com/ClickHouse/ClickHouse/pull/67791) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Integration tests: fix flaky `test_dictionaries_update_and_reload::test_reload_after_fail_by_timer`. [#67793](https://github.com/ClickHouse/ClickHouse/pull/67793) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Fix possible CANNOT_READ_ALL_DATA during server startup in performance tests. [#67795](https://github.com/ClickHouse/ClickHouse/pull/67795) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Reduce table size in 03037_dynamic_merges_2* tests. [#67797](https://github.com/ClickHouse/ClickHouse/pull/67797) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Disable 03038_nested_dynamic_merges* under sanitizers because it's too slow. [#67798](https://github.com/ClickHouse/ClickHouse/pull/67798) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Revert "Merge pull request [#66510](https://github.com/ClickHouse/ClickHouse/issues/66510) from canhld94/fix_trivial_count_non_deterministic_func". [#67800](https://github.com/ClickHouse/ClickHouse/pull/67800) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Update comment. [#67801](https://github.com/ClickHouse/ClickHouse/pull/67801) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix bad test `03032_redundant_equals`. [#67822](https://github.com/ClickHouse/ClickHouse/pull/67822) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Update tags for a few tests. [#67829](https://github.com/ClickHouse/ClickHouse/pull/67829) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Add debug logging for window view tests. [#67841](https://github.com/ClickHouse/ClickHouse/pull/67841) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Closes [#67621](https://github.com/ClickHouse/ClickHouse/issues/67621). [#67843](https://github.com/ClickHouse/ClickHouse/pull/67843) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Fix query cache randomization in stress tests. [#67855](https://github.com/ClickHouse/ClickHouse/pull/67855) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Update version_date.tsv and changelogs after v24.5.5.78-stable. [#67863](https://github.com/ClickHouse/ClickHouse/pull/67863) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Un-flake 02524_fuzz_and_fuss_2. [#67867](https://github.com/ClickHouse/ClickHouse/pull/67867) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Misc fixes. [#67869](https://github.com/ClickHouse/ClickHouse/pull/67869) ([Alexey Katsman](https://github.com/alexkats)).
|
||||
* Fixes [#67444](https://github.com/ClickHouse/ClickHouse/issues/67444). [#67873](https://github.com/ClickHouse/ClickHouse/pull/67873) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* no-msan 00314_sample_factor_virtual_column. [#67874](https://github.com/ClickHouse/ClickHouse/pull/67874) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Revert "Revert "Add a test for [#47892](https://github.com/ClickHouse/ClickHouse/issues/47892)"". [#67877](https://github.com/ClickHouse/ClickHouse/pull/67877) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Lazily create invalid files in S3. [#67882](https://github.com/ClickHouse/ClickHouse/pull/67882) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Do not try to create azure container if not needed. [#67896](https://github.com/ClickHouse/ClickHouse/pull/67896) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* CI: Fix for setting Mergeable Check from sync. [#67898](https://github.com/ClickHouse/ClickHouse/pull/67898) ([Max K.](https://github.com/maxknv)).
|
||||
* Bump rocksdb from v8.10 to v9.4 + enable jemalloc and liburing. [#67904](https://github.com/ClickHouse/ClickHouse/pull/67904) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update version_date.tsv and changelogs after v24.6.3.95-stable. [#67910](https://github.com/ClickHouse/ClickHouse/pull/67910) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Remove some no-parallel tags from tests (Part 3). [#67914](https://github.com/ClickHouse/ClickHouse/pull/67914) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Follow up [#67235](https://github.com/ClickHouse/ClickHouse/issues/67235). [#67917](https://github.com/ClickHouse/ClickHouse/pull/67917) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* CI: Changelog: Critical Bug Fix to Bug Fix. [#67919](https://github.com/ClickHouse/ClickHouse/pull/67919) ([Max K.](https://github.com/maxknv)).
|
||||
* CI: Multi-channel CiBuddy. [#67923](https://github.com/ClickHouse/ClickHouse/pull/67923) ([Max K.](https://github.com/maxknv)).
|
||||
* more logs to debug logical error from async inserts. [#67928](https://github.com/ClickHouse/ClickHouse/pull/67928) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Fix stress test error with TDigest statistics. [#67930](https://github.com/ClickHouse/ClickHouse/pull/67930) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Remove some no-parallel tags from tests (Part 4). [#67932](https://github.com/ClickHouse/ClickHouse/pull/67932) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Upgrade QPL to v1.6.0. [#67933](https://github.com/ClickHouse/ClickHouse/pull/67933) ([Maria Zhukova](https://github.com/mzhukova)).
|
||||
* CI: Strict job timeout 1.5h for tests, 2h for builds. [#67934](https://github.com/ClickHouse/ClickHouse/pull/67934) ([Max K.](https://github.com/maxknv)).
|
||||
* Remove slow tests from fasttest check. [#67941](https://github.com/ClickHouse/ClickHouse/pull/67941) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix memory corruption in usearch. [#67942](https://github.com/ClickHouse/ClickHouse/pull/67942) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Backported in [#68547](https://github.com/ClickHouse/ClickHouse/issues/68547): Timeout handling for functional and integration tests, store artifacts and report if timed out - sets 2h default timeout for all jobs. [#67944](https://github.com/ClickHouse/ClickHouse/pull/67944) ([Max K.](https://github.com/maxknv)).
|
||||
* Unflake 02099_tsv_raw_format.sh. [#67947](https://github.com/ClickHouse/ClickHouse/pull/67947) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* This closes: [#67866](https://github.com/ClickHouse/ClickHouse/issues/67866). [#67950](https://github.com/ClickHouse/ClickHouse/pull/67950) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Change log level of an insignificant message in clickhouse-local. [#67952](https://github.com/ClickHouse/ClickHouse/pull/67952) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add an explicit error for `ALTER MODIFY SQL SECURITY` on non-view tables. [#67953](https://github.com/ClickHouse/ClickHouse/pull/67953) ([pufit](https://github.com/pufit)).
|
||||
* Fix flaky `test_storage_s3_queue/test.py::test_multiple_tables_streaming_sync_distributed`. [#67959](https://github.com/ClickHouse/ClickHouse/pull/67959) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* tests: fix 03002_part_log_rmt_fetch_merge_error flakiness. [#67960](https://github.com/ClickHouse/ClickHouse/pull/67960) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix timeout of 02310_clickhouse_local_INSERT_progress_profile_events. [#67961](https://github.com/ClickHouse/ClickHouse/pull/67961) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. III. [#67964](https://github.com/ClickHouse/ClickHouse/pull/67964) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update minio in stateless tests. [#67975](https://github.com/ClickHouse/ClickHouse/pull/67975) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* CI: Integration tests uncover some logging. [#67978](https://github.com/ClickHouse/ClickHouse/pull/67978) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix 03130_convert_outer_join_to_inner_join. [#67980](https://github.com/ClickHouse/ClickHouse/pull/67980) ([vdimir](https://github.com/vdimir)).
|
||||
* Collect minio audit logs in stateless tests. [#67998](https://github.com/ClickHouse/ClickHouse/pull/67998) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Remove some no-parallel tags from tests (Part 5). [#68002](https://github.com/ClickHouse/ClickHouse/pull/68002) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Minor fixes in tables.md. [#68004](https://github.com/ClickHouse/ClickHouse/pull/68004) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Follow up for [#67843](https://github.com/ClickHouse/ClickHouse/issues/67843). [#68007](https://github.com/ClickHouse/ClickHouse/pull/68007) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Remove unused CLI option. [#68008](https://github.com/ClickHouse/ClickHouse/pull/68008) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `02845_threads_count_in_distributed_queries`. [#68011](https://github.com/ClickHouse/ClickHouse/pull/68011) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* CI: Pass job timeout into tests config. [#68013](https://github.com/ClickHouse/ClickHouse/pull/68013) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Add a test for [#57420](https://github.com/ClickHouse/ClickHouse/issues/57420). [#68017](https://github.com/ClickHouse/ClickHouse/pull/68017) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Revert "Revert "Bump rocksdb from v8.10 to v9.4 + enable jemalloc and liburing"". [#68021](https://github.com/ClickHouse/ClickHouse/pull/68021) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* CI: Fix for filtering jobs in PRs. [#68022](https://github.com/ClickHouse/ClickHouse/pull/68022) ([Max K.](https://github.com/maxknv)).
|
||||
* Docs: Update 3rd party library guide. [#68027](https://github.com/ClickHouse/ClickHouse/pull/68027) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Refactor tests for (experimental) statistics. [#68034](https://github.com/ClickHouse/ClickHouse/pull/68034) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Split `00284_external_aggregation.sql`. [#68037](https://github.com/ClickHouse/ClickHouse/pull/68037) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update version_date.tsv and changelog after v24.7.3.42-stable. [#68045](https://github.com/ClickHouse/ClickHouse/pull/68045) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update test_drop_is_lock_free/test.py. [#68051](https://github.com/ClickHouse/ClickHouse/pull/68051) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Fixes [#67865](https://github.com/ClickHouse/ClickHouse/issues/67865). [#68054](https://github.com/ClickHouse/ClickHouse/pull/68054) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Disable randomization of `trace_profile_events` in clickhouse-test. [#68058](https://github.com/ClickHouse/ClickHouse/pull/68058) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Minor CMake cleanup. [#68069](https://github.com/ClickHouse/ClickHouse/pull/68069) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* If the test cluster is overloaded, sometimes simple query execution [can take more time](https://pastila.nl/?00224e71/f017cd6675b52ccc205c81aa62a47de5#8dB4+C4MOdOi3NLV1dc0Fg==) than `Buffer`'s max time to flush. This PR doubles the timeout and allows to skip the check in case of significant latency. [#68072](https://github.com/ClickHouse/ClickHouse/pull/68072) ([pufit](https://github.com/pufit)).
|
||||
* Fix flaky `02675_profile_events_from_query_log_and_client`. [#68097](https://github.com/ClickHouse/ClickHouse/pull/68097) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix race in `WithRetries`. [#68106](https://github.com/ClickHouse/ClickHouse/pull/68106) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Add empty cell to reports when time is missing. [#68112](https://github.com/ClickHouse/ClickHouse/pull/68112) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Fix test `00900_long_parquet_load`. [#68130](https://github.com/ClickHouse/ClickHouse/pull/68130) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* tests: fix 01246_buffer_flush flakiness due to slow trace_log flush. [#68134](https://github.com/ClickHouse/ClickHouse/pull/68134) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Only use Field::safeGet - Field::get prone to type punning. [#68135](https://github.com/ClickHouse/ClickHouse/pull/68135) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* tests: attempt to fix 01600_parts_states_metrics_long (by forbid parallel run). [#68136](https://github.com/ClickHouse/ClickHouse/pull/68136) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix01710 Timeout. [#68138](https://github.com/ClickHouse/ClickHouse/pull/68138) ([jsc0218](https://github.com/jsc0218)).
|
||||
* Remove the extra cell from reports when it is not necessary. [#68145](https://github.com/ClickHouse/ClickHouse/pull/68145) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Remove "Processing configuration file" message from clickhouse-local. [#68157](https://github.com/ClickHouse/ClickHouse/pull/68157) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* tests: fix 02122_join_group_by_timeout flakiness. [#68160](https://github.com/ClickHouse/ClickHouse/pull/68160) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix `test_cluster_all_replicas`. [#68178](https://github.com/ClickHouse/ClickHouse/pull/68178) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix leftovers. [#68181](https://github.com/ClickHouse/ClickHouse/pull/68181) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `01172_transaction_counters`. [#68182](https://github.com/ClickHouse/ClickHouse/pull/68182) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Refactor tests for (experimental) statistics. [#68186](https://github.com/ClickHouse/ClickHouse/pull/68186) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Remove Log engine from Kafka integration tests. [#68200](https://github.com/ClickHouse/ClickHouse/pull/68200) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* [Green CI] Fix test 01903_correct_block_size_prediction_with_default. [#68203](https://github.com/ClickHouse/ClickHouse/pull/68203) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Replace segfault in Replicated database with logical error. [#68250](https://github.com/ClickHouse/ClickHouse/pull/68250) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#68423](https://github.com/ClickHouse/ClickHouse/issues/68423): tests: make 01600_parts_states_metrics_long better. [#68265](https://github.com/ClickHouse/ClickHouse/pull/68265) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Backported in [#68374](https://github.com/ClickHouse/ClickHouse/issues/68374): Rename: S3DiskNoKeyErrors -> DiskS3NoSuchKeyErrors. [#68361](https://github.com/ClickHouse/ClickHouse/pull/68361) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#68637](https://github.com/ClickHouse/ClickHouse/issues/68637): Check for invalid regexp in JSON SKIP REGEXP section. [#68451](https://github.com/ClickHouse/ClickHouse/pull/68451) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#68485](https://github.com/ClickHouse/ClickHouse/issues/68485): Better inference of date times 2. [#68452](https://github.com/ClickHouse/ClickHouse/pull/68452) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#68540](https://github.com/ClickHouse/ClickHouse/issues/68540): CI: Native build for package_aarch64. [#68457](https://github.com/ClickHouse/ClickHouse/pull/68457) ([Max K.](https://github.com/maxknv)).
|
||||
* Backported in [#68518](https://github.com/ClickHouse/ClickHouse/issues/68518): Minor update in Dynamic/JSON serializations. [#68459](https://github.com/ClickHouse/ClickHouse/pull/68459) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#68558](https://github.com/ClickHouse/ClickHouse/issues/68558): CI: Minor release workflow fix. [#68536](https://github.com/ClickHouse/ClickHouse/pull/68536) ([Max K.](https://github.com/maxknv)).
|
||||
* Backported in [#68576](https://github.com/ClickHouse/ClickHouse/issues/68576): CI: Tidy build timeout from 2h to 3h. [#68567](https://github.com/ClickHouse/ClickHouse/pull/68567) ([Max K.](https://github.com/maxknv)).
|
||||
|
||||
@ -91,6 +91,28 @@ SELECT 1
|
||||
In addition to the above settings, you can use `USE_*` flags from `system.build_options` to define usage of particular ClickHouse features.
|
||||
For example, if your test uses a MySQL table, you should add a tag `use-mysql`.
|
||||
|
||||
### Specifying limits for random settings
|
||||
|
||||
A test can specify minimum and maximum allowed values for settings that can be randomized during test run.
|
||||
|
||||
For `.sh` tests limits are written as a comment on the line next to tags or on the second line if no tags are specified:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest
|
||||
# Random settings limits: max_block_size=(1000, 10000); index_granularity=(100, None)
|
||||
```
|
||||
|
||||
For `.sql` tests tags are placed as a SQL comment in the line next to tags or in the first line:
|
||||
|
||||
```sql
|
||||
-- Tags: no-fasttest
|
||||
-- Random settings limits: max_block_size=(1000, 10000); index_granularity=(100, None)
|
||||
SELECT 1
|
||||
```
|
||||
|
||||
If you need to specify only one limit, you can use `None` for another one.
|
||||
|
||||
### Choosing the Test Name
|
||||
|
||||
The name of the test starts with a five-digit prefix followed by a descriptive name, such as `00422_hash_function_constexpr.sql`. To choose the prefix, find the largest prefix already present in the directory, and increment it by one. In the meantime, some other tests might be added with the same numeric prefix, but this is OK and does not lead to any problems, you don't have to change it later.
|
||||
|
||||
@ -240,7 +240,7 @@ libhdfs3 support HDFS namenode HA.
|
||||
## Storage Settings {#storage-settings}
|
||||
|
||||
- [hdfs_truncate_on_insert](/docs/en/operations/settings/settings.md#hdfs_truncate_on_insert) - allows to truncate file before insert into it. Disabled by default.
|
||||
- [hdfs_create_multiple_files](/docs/en/operations/settings/settings.md#hdfs_allow_create_multiple_files) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [hdfs_create_new_file_on_insert](/docs/en/operations/settings/settings.md#hdfs_create_new_file_on_insert) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [hdfs_skip_empty_files](/docs/en/operations/settings/settings.md#hdfs_skip_empty_files) - allows to skip empty files while reading. Disabled by default.
|
||||
|
||||
**See Also**
|
||||
|
||||
@ -225,7 +225,7 @@ CREATE TABLE table_with_asterisk (name String, value UInt32)
|
||||
## Storage Settings {#storage-settings}
|
||||
|
||||
- [s3_truncate_on_insert](/docs/en/operations/settings/settings.md#s3_truncate_on_insert) - allows to truncate file before insert into it. Disabled by default.
|
||||
- [s3_create_multiple_files](/docs/en/operations/settings/settings.md#s3_allow_create_multiple_files) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [s3_create_new_file_on_insert](/docs/en/operations/settings/settings.md#s3_create_new_file_on_insert) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [s3_skip_empty_files](/docs/en/operations/settings/settings.md#s3_skip_empty_files) - allows to skip empty files while reading. Disabled by default.
|
||||
|
||||
## S3-related Settings {#settings}
|
||||
|
||||
@ -73,13 +73,21 @@ In the above example the `password_sha256_hex` value is the hexadecimal represen
|
||||
|
||||
### Storage for named collections
|
||||
|
||||
Named collections can either be stored on local disk or in zookeeper/keeper. By default local storage is used.
|
||||
Named collections can either be stored on local disk or in ZooKeeper/Keeper. By default local storage is used.
|
||||
They can also be stored using encryption with the same algorithms used for [disk encryption](storing-data#encrypted-virtual-file-system),
|
||||
where `aes_128_ctr` is used by default.
|
||||
|
||||
To configure named collections storage in keeper and a `type` (equal to either `keeper` or `zookeeper`) and `path` (path in keeper, where named collections will be stored) to `named_collections_storage` section in configuration file:
|
||||
To configure named collections storage you need to specify a `type`. This can be either `local` or `keeper`/`zookeeper`. For encrypted storage,
|
||||
you can use `local_encrypted` or `keeper_encrypted`/`zookeeper_encrypted`.
|
||||
|
||||
To use ZooKeeper/Keeper we also need to set up a `path` (path in ZooKeeper/Keeper, where named collections will be stored) to
|
||||
`named_collections_storage` section in configuration file. The following example uses encryption and ZooKeeper/Keeper:
|
||||
```
|
||||
<clickhouse>
|
||||
<named_collections_storage>
|
||||
<type>zookeeper</type>
|
||||
<type>zookeeper_encrypted</type>
|
||||
<key_hex>bebec0cabebec0cabebec0cabebec0ca</key_hex>
|
||||
<algorithm>aes_128_ctr</algorithm>
|
||||
<path>/named_collections_path/</path>
|
||||
<update_timeout_ms>1000</update_timeout_ms>
|
||||
</named_collections_storage>
|
||||
|
||||
@ -155,6 +155,8 @@ SELECT 1 SETTINGS use_query_cache = true, query_cache_tag = 'tag 1';
|
||||
SELECT 1 SETTINGS use_query_cache = true, query_cache_tag = 'tag 2';
|
||||
```
|
||||
|
||||
To remove only entries with tag `tag` from the query cache, you can use statement `SYSTEM DROP QUERY CACHE TAG 'tag'`.
|
||||
|
||||
ClickHouse reads table data in blocks of [max_block_size](settings/settings.md#setting-max_block_size) rows. Due to filtering, aggregation,
|
||||
etc., result blocks are typically much smaller than 'max_block_size' but there are also cases where they are much bigger. Setting
|
||||
[query_cache_squash_partial_results](settings/settings.md#query-cache-squash-partial-results) (enabled by default) controls if result blocks
|
||||
|
||||
@ -194,6 +194,17 @@ If enabled, ClickHouse will try to infer type `DateTime64` from string fields in
|
||||
|
||||
Enabled by default.
|
||||
|
||||
## input_format_try_infer_variants {#input_format_try_infer_variants}
|
||||
|
||||
If enabled, ClickHouse will try to infer type [`Variant`](../../sql-reference/data-types/variant.md) in schema inference for text formats when there is more than one possible type for column/array elements.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 — Disabled.
|
||||
- 1 — Enabled.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
## date_time_input_format {#date_time_input_format}
|
||||
|
||||
Allows choosing a parser of the text representation of date and time.
|
||||
|
||||
@ -2855,7 +2855,7 @@ The minimum chunk size in bytes, which each thread will parse in parallel.
|
||||
|
||||
## merge_selecting_sleep_ms {#merge_selecting_sleep_ms}
|
||||
|
||||
Sleep time for merge selecting when no part is selected. A lower setting triggers selecting tasks in `background_schedule_pool` frequently, which results in a large number of requests to ClickHouse Keeper in large-scale clusters.
|
||||
Minimum time to wait before trying to select parts to merge again after no parts were selected. A lower setting triggers selecting tasks in `background_schedule_pool` frequently, which results in a large number of requests to ClickHouse Keeper in large-scale clusters.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -2863,6 +2863,16 @@ Possible values:
|
||||
|
||||
Default value: `5000`.
|
||||
|
||||
## max_merge_selecting_sleep_ms
|
||||
|
||||
Maximum time to wait before trying to select parts to merge again after no parts were selected. A lower setting triggers selecting tasks in `background_schedule_pool` frequently, which results in a large number of requests to ClickHouse Keeper in large-scale clusters.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: `60000`.
|
||||
|
||||
## parallel_distributed_insert_select {#parallel_distributed_insert_select}
|
||||
|
||||
Enables parallel distributed `INSERT ... SELECT` query.
|
||||
|
||||
@ -53,29 +53,28 @@ SELECT now() as current_date_time, current_date_time + INTERVAL 4 DAY
|
||||
└─────────────────────┴───────────────────────────────┘
|
||||
```
|
||||
|
||||
Intervals with different types can’t be combined. You can’t use intervals like `4 DAY 1 HOUR`. Specify intervals in units that are smaller or equal to the smallest unit of the interval, for example, the interval `1 day and an hour` interval can be expressed as `25 HOUR` or `90000 SECOND`.
|
||||
|
||||
You can’t perform arithmetical operations with `Interval`-type values, but you can add intervals of different types consequently to values in `Date` or `DateTime` data types. For example:
|
||||
Also it is possible to use multiple intervals simultaneously:
|
||||
|
||||
``` sql
|
||||
SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR
|
||||
SELECT now() AS current_date_time, current_date_time + (INTERVAL 4 DAY + INTERVAL 3 HOUR)
|
||||
```
|
||||
|
||||
``` text
|
||||
┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐
|
||||
│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │
|
||||
└─────────────────────┴────────────────────────────────────────────────────────┘
|
||||
┌───current_date_time─┬─plus(current_date_time, plus(toIntervalDay(4), toIntervalHour(3)))─┐
|
||||
│ 2024-08-08 18:31:39 │ 2024-08-12 21:31:39 │
|
||||
└─────────────────────┴────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
The following query causes an exception:
|
||||
And to compare values with different intervals:
|
||||
|
||||
``` sql
|
||||
select now() AS current_date_time, current_date_time + (INTERVAL 4 DAY + INTERVAL 3 HOUR)
|
||||
SELECT toIntervalMicrosecond(3600000000) = toIntervalHour(1);
|
||||
```
|
||||
|
||||
``` text
|
||||
Received exception from server (version 19.14.1):
|
||||
Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argument types for function plus: if one argument is Interval, then another must be Date or DateTime..
|
||||
┌─less(toIntervalMicrosecond(179999999), toIntervalMinute(3))─┐
|
||||
│ 1 │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## See Also
|
||||
|
||||
@ -8,6 +8,78 @@ sidebar_label: Replacing in Strings
|
||||
|
||||
[General strings functions](string-functions.md) and [functions for searching in strings](string-search-functions.md) are described separately.
|
||||
|
||||
## overlay
|
||||
|
||||
Replace part of the string `input` with another string `replace`, starting at the 1-based index `offset`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
overlay(s, replace, offset[, length])
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `input`: A string type [String](../data-types/string.md).
|
||||
- `replace`: A string type [String](../data-types/string.md).
|
||||
- `offset`: An integer type [Int](../data-types/int-uint.md). If `offset` is negative, it is counted from the end of the `input` string.
|
||||
- `length`: Optional. An integer type [Int](../data-types/int-uint.md). `length` specifies the length of the snippet within input to be replaced. If `length` is not specified, the number of bytes removed from `input` equals the length of `replace`; otherwise `length` bytes are removed.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- A [String](../data-types/string.md) data type value.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT overlay('ClickHouse SQL', 'CORE', 12) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─res─────────────┐
|
||||
│ ClickHouse CORE │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
## overlayUTF8
|
||||
|
||||
Replace part of the string `input` with another string `replace`, starting at the 1-based index `offset`.
|
||||
|
||||
Assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
overlayUTF8(s, replace, offset[, length])
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `s`: A string type [String](../data-types/string.md).
|
||||
- `replace`: A string type [String](../data-types/string.md).
|
||||
- `offset`: An integer type [Int](../data-types/int-uint.md). If `offset` is negative, it is counted from the end of the `input` string.
|
||||
- `length`: Optional. An integer type [Int](../data-types/int-uint.md). `length` specifies the length of the snippet within input to be replaced. If `length` is not specified, the number of characters removed from `input` equals the length of `replace`; otherwise `length` characters are removed.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- A [String](../data-types/string.md) data type value.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT overlayUTF8('ClickHouse是一款OLAP数据库', '开源', 12, 2) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─res────────────────────────┐
|
||||
│ ClickHouse是开源OLAP数据库 │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## replaceOne
|
||||
|
||||
Replaces the first occurrence of the substring `pattern` in `haystack` by the `replacement` string.
|
||||
|
||||
@ -136,7 +136,13 @@ The compiled expression cache is enabled/disabled with the query/user/profile-le
|
||||
|
||||
## DROP QUERY CACHE
|
||||
|
||||
```sql
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
SYSTEM DROP QUERY CACHE TAG '<tag>'
|
||||
````
|
||||
|
||||
Clears the [query cache](../../operations/query-cache.md).
|
||||
If a tag is specified, only query cache entries with the specified tag are deleted.
|
||||
|
||||
## DROP FORMAT SCHEMA CACHE {#system-drop-schema-format}
|
||||
|
||||
|
||||
@ -116,7 +116,7 @@ SELECT * from HDFS('hdfs://hdfs1:9000/data/path/date=*/country=*/code=*/*.parque
|
||||
## Storage Settings {#storage-settings}
|
||||
|
||||
- [hdfs_truncate_on_insert](/docs/en/operations/settings/settings.md#hdfs_truncate_on_insert) - allows to truncate file before insert into it. Disabled by default.
|
||||
- [hdfs_create_multiple_files](/docs/en/operations/settings/settings.md#hdfs_allow_create_multiple_files) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [hdfs_create_new_file_on_insert](/docs/en/operations/settings/settings.md#hdfs_create_new_file_on_insert) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [hdfs_skip_empty_files](/docs/en/operations/settings/settings.md#hdfs_skip_empty_files) - allows to skip empty files while reading. Disabled by default.
|
||||
- [ignore_access_denied_multidirectory_globs](/docs/en/operations/settings/settings.md#ignore_access_denied_multidirectory_globs) - allows to ignore permission denied errors for multi-directory globs.
|
||||
|
||||
|
||||
@ -290,7 +290,7 @@ SELECT * from s3('s3://data/path/date=*/country=*/code=*/*.parquet') where _date
|
||||
## Storage Settings {#storage-settings}
|
||||
|
||||
- [s3_truncate_on_insert](/docs/en/operations/settings/settings.md#s3_truncate_on_insert) - allows to truncate file before insert into it. Disabled by default.
|
||||
- [s3_create_multiple_files](/docs/en/operations/settings/settings.md#s3_allow_create_multiple_files) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [s3_create_new_file_on_insert](/docs/en/operations/settings/settings.md#s3_create_new_file_on_insert) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [s3_skip_empty_files](/docs/en/operations/settings/settings.md#s3_skip_empty_files) - allows to skip empty files while reading. Disabled by default.
|
||||
|
||||
**See Also**
|
||||
|
||||
@ -54,29 +54,28 @@ SELECT now() as current_date_time, current_date_time + INTERVAL 4 DAY
|
||||
└─────────────────────┴───────────────────────────────┘
|
||||
```
|
||||
|
||||
Нельзя объединять интервалы различных типов. Нельзя использовать интервалы вида `4 DAY 1 HOUR`. Вместо этого выражайте интервал в единицах меньших или равных минимальной единице интервала, например, интервал «1 день и 1 час» можно выразить как `25 HOUR` или `90000 SECOND`.
|
||||
|
||||
Арифметические операции со значениями типов `Interval` не доступны, однако можно последовательно добавлять различные интервалы к значениям типов `Date` и `DateTime`. Например:
|
||||
Также можно использовать различные типы интервалов одновременно:
|
||||
|
||||
``` sql
|
||||
SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR
|
||||
SELECT now() AS current_date_time, current_date_time + (INTERVAL 4 DAY + INTERVAL 3 HOUR)
|
||||
```
|
||||
|
||||
``` text
|
||||
┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐
|
||||
│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │
|
||||
└─────────────────────┴────────────────────────────────────────────────────────┘
|
||||
┌───current_date_time─┬─plus(current_date_time, plus(toIntervalDay(4), toIntervalHour(3)))─┐
|
||||
│ 2024-08-08 18:31:39 │ 2024-08-12 21:31:39 │
|
||||
└─────────────────────┴────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Следующий запрос приведёт к генерированию исключения:
|
||||
И сравнивать значения из разными интервалами:
|
||||
|
||||
``` sql
|
||||
select now() AS current_date_time, current_date_time + (INTERVAL 4 DAY + INTERVAL 3 HOUR)
|
||||
SELECT toIntervalMicrosecond(3600000000) = toIntervalHour(1);
|
||||
```
|
||||
|
||||
``` text
|
||||
Received exception from server (version 19.14.1):
|
||||
Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argument types for function plus: if one argument is Interval, then another must be Date or DateTime..
|
||||
┌─less(toIntervalMicrosecond(179999999), toIntervalMinute(3))─┐
|
||||
│ 1 │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Смотрите также {#smotrite-takzhe}
|
||||
|
||||
@ -1164,9 +1164,6 @@ void Client::processOptions(const OptionsDescription & options_description,
|
||||
/// (There is no need to copy the context because clickhouse-client has no background tasks so it won't use that context in parallel.)
|
||||
client_context = global_context;
|
||||
initClientContext();
|
||||
|
||||
/// Allow to pass-through unknown settings to the server.
|
||||
client_context->getAccessControl().allowAllSettings();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1922,7 +1922,7 @@ try
|
||||
auto & access_control = global_context->getAccessControl();
|
||||
try
|
||||
{
|
||||
access_control.setupFromMainConfig(config(), config_path, [&] { return global_context->getZooKeeper(); });
|
||||
access_control.setUpFromMainConfig(config(), config_path, [&] { return global_context->getZooKeeper(); });
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
|
||||
@ -280,7 +280,7 @@ void AccessControl::shutdown()
|
||||
}
|
||||
|
||||
|
||||
void AccessControl::setupFromMainConfig(const Poco::Util::AbstractConfiguration & config_, const String & config_path_,
|
||||
void AccessControl::setUpFromMainConfig(const Poco::Util::AbstractConfiguration & config_, const String & config_path_,
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_)
|
||||
{
|
||||
if (config_.has("custom_settings_prefixes"))
|
||||
@ -868,10 +868,4 @@ const ExternalAuthenticators & AccessControl::getExternalAuthenticators() const
|
||||
return *external_authenticators;
|
||||
}
|
||||
|
||||
|
||||
void AccessControl::allowAllSettings()
|
||||
{
|
||||
custom_settings_prefixes->registerPrefixes({""});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -57,7 +57,7 @@ public:
|
||||
void shutdown() override;
|
||||
|
||||
/// Initializes access storage (user directories).
|
||||
void setupFromMainConfig(const Poco::Util::AbstractConfiguration & config_, const String & config_path_,
|
||||
void setUpFromMainConfig(const Poco::Util::AbstractConfiguration & config_, const String & config_path_,
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_);
|
||||
|
||||
/// Parses access entities from a configuration loaded from users.xml.
|
||||
@ -238,9 +238,6 @@ public:
|
||||
/// Gets manager of notifications.
|
||||
AccessChangesNotifier & getChangesNotifier();
|
||||
|
||||
/// Allow all setting names - this can be used in clients to pass-through unknown settings to the server.
|
||||
void allowAllSettings();
|
||||
|
||||
private:
|
||||
class ContextAccessCache;
|
||||
class CustomSettingsPrefixes;
|
||||
|
||||
@ -219,8 +219,8 @@ void SettingsConstraints::clamp(const Settings & current_settings, SettingsChang
|
||||
});
|
||||
}
|
||||
|
||||
template <typename SettingsT>
|
||||
bool getNewValueToCheck(const SettingsT & current_settings, SettingChange & change, Field & new_value, bool throw_on_failure)
|
||||
template <class T>
|
||||
bool getNewValueToCheck(const T & current_settings, SettingChange & change, Field & new_value, bool throw_on_failure)
|
||||
{
|
||||
Field current_value;
|
||||
bool has_current_value = current_settings.tryGet(change.name, current_value);
|
||||
@ -230,12 +230,12 @@ bool getNewValueToCheck(const SettingsT & current_settings, SettingChange & chan
|
||||
return false;
|
||||
|
||||
if (throw_on_failure)
|
||||
new_value = SettingsT::castValueUtil(change.name, change.value);
|
||||
new_value = T::castValueUtil(change.name, change.value);
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
new_value = SettingsT::castValueUtil(change.name, change.value);
|
||||
new_value = T::castValueUtil(change.name, change.value);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
|
||||
@ -177,9 +177,10 @@ ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const
|
||||
* It could also lead to ambiguous parsing because we don't know if the string literal represents a date or a Decimal64 literal.
|
||||
* For this reason, we use a string literal representing a date instead of a Decimal64 literal.
|
||||
*/
|
||||
if (WhichDataType(constant_value_type->getTypeId()).isDateTime64())
|
||||
const auto & constant_value_end_type = removeNullable(constant_value_type); /// if Nullable
|
||||
if (WhichDataType(constant_value_end_type->getTypeId()).isDateTime64())
|
||||
{
|
||||
const auto * date_time_type = typeid_cast<const DataTypeDateTime64 *>(constant_value_type.get());
|
||||
const auto * date_time_type = typeid_cast<const DataTypeDateTime64 *>(constant_value_end_type.get());
|
||||
DecimalField<Decimal64> decimal_value;
|
||||
if (constant_value_literal.tryGet<DecimalField<Decimal64>>(decimal_value))
|
||||
{
|
||||
|
||||
@ -58,7 +58,6 @@
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
#include <Interpreters/ReplaceQueryParameterVisitor.h>
|
||||
#include <Interpreters/ProfileEventsExt.h>
|
||||
#include <Interpreters/InterpreterSetQuery.h>
|
||||
#include <IO/WriteBufferFromOStream.h>
|
||||
#include <IO/WriteBufferFromFileDescriptor.h>
|
||||
#include <IO/CompressionMethod.h>
|
||||
@ -1893,19 +1892,48 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
|
||||
profile_events.watch.restart();
|
||||
|
||||
{
|
||||
/// Temporarily apply query settings to the context.
|
||||
Settings old_settings = client_context->getSettingsCopy();
|
||||
SCOPE_EXIT_SAFE(
|
||||
{
|
||||
client_context->setSettings(old_settings);
|
||||
/// Temporarily apply query settings to context.
|
||||
std::optional<Settings> old_settings;
|
||||
SCOPE_EXIT_SAFE({
|
||||
if (old_settings)
|
||||
client_context->setSettings(*old_settings);
|
||||
});
|
||||
InterpreterSetQuery::applySettingsFromQuery(parsed_query, client_context);
|
||||
|
||||
auto apply_query_settings = [&](const IAST & settings_ast)
|
||||
{
|
||||
if (!old_settings)
|
||||
old_settings.emplace(client_context->getSettingsRef());
|
||||
client_context->applySettingsChanges(settings_ast.as<ASTSetQuery>()->changes);
|
||||
client_context->resetSettingsToDefaultValue(settings_ast.as<ASTSetQuery>()->default_settings);
|
||||
};
|
||||
|
||||
const auto * insert = parsed_query->as<ASTInsertQuery>();
|
||||
if (const auto * select = parsed_query->as<ASTSelectQuery>(); select && select->settings())
|
||||
apply_query_settings(*select->settings());
|
||||
else if (const auto * select_with_union = parsed_query->as<ASTSelectWithUnionQuery>())
|
||||
{
|
||||
const ASTs & children = select_with_union->list_of_selects->children;
|
||||
if (!children.empty())
|
||||
{
|
||||
// On the client it is enough to apply settings only for the
|
||||
// last SELECT, since the only thing that is important to apply
|
||||
// on the client is format settings.
|
||||
const auto * last_select = children.back()->as<ASTSelectQuery>();
|
||||
if (last_select && last_select->settings())
|
||||
{
|
||||
apply_query_settings(*last_select->settings());
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (const auto * query_with_output = parsed_query->as<ASTQueryWithOutput>(); query_with_output && query_with_output->settings_ast)
|
||||
apply_query_settings(*query_with_output->settings_ast);
|
||||
else if (insert && insert->settings_ast)
|
||||
apply_query_settings(*insert->settings_ast);
|
||||
|
||||
if (!connection->checkConnected(connection_parameters.timeouts))
|
||||
connect();
|
||||
|
||||
ASTPtr input_function;
|
||||
const auto * insert = parsed_query->as<ASTInsertQuery>();
|
||||
if (insert && insert->select)
|
||||
insert->tryFindInputFunction(input_function);
|
||||
|
||||
|
||||
@ -197,6 +197,12 @@ public:
|
||||
cache_policy->remove(key);
|
||||
}
|
||||
|
||||
void remove(std::function<bool(const Key&, const MappedPtr &)> predicate)
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
cache_policy->remove(predicate);
|
||||
}
|
||||
|
||||
size_t sizeInBytes() const
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
@ -55,6 +55,7 @@ public:
|
||||
virtual void set(const Key & key, const MappedPtr & mapped) = 0;
|
||||
|
||||
virtual void remove(const Key & key) = 0;
|
||||
virtual void remove(std::function<bool(const Key & key, const MappedPtr & mapped)> predicate) = 0;
|
||||
|
||||
virtual void clear() = 0;
|
||||
virtual std::vector<KeyMapped> dump() const = 0;
|
||||
|
||||
@ -79,6 +79,22 @@ public:
|
||||
cells.erase(it);
|
||||
}
|
||||
|
||||
void remove(std::function<bool(const Key &, const MappedPtr &)> predicate) override
|
||||
{
|
||||
for (auto it = cells.begin(); it != cells.end();)
|
||||
{
|
||||
if (predicate(it->first, it->second.value))
|
||||
{
|
||||
Cell & cell = it->second;
|
||||
current_size_in_bytes -= cell.size;
|
||||
queue.erase(cell.queue_iterator);
|
||||
it = cells.erase(it);
|
||||
}
|
||||
else
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
MappedPtr get(const Key & key) override
|
||||
{
|
||||
auto it = cells.find(key);
|
||||
|
||||
@ -6,14 +6,18 @@
|
||||
#include <Common/ZooKeeper/KeeperException.h>
|
||||
#include <Common/ZooKeeper/ZooKeeper.h>
|
||||
#include <Core/Settings.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <IO/FileEncryptionCommon.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <Parsers/ParserCreateQuery.h>
|
||||
#include <Parsers/formatAST.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <filesystem>
|
||||
#include <boost/algorithm/hex.hpp>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
@ -26,6 +30,7 @@ namespace ErrorCodes
|
||||
extern const int INVALID_CONFIG_PARAMETER;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
}
|
||||
|
||||
static const std::string named_collections_storage_config_path = "named_collections_storage";
|
||||
@ -74,9 +79,9 @@ public:
|
||||
};
|
||||
|
||||
|
||||
class NamedCollectionsMetadataStorage::LocalStorage : public INamedCollectionsStorage, private WithContext
|
||||
class NamedCollectionsMetadataStorage::LocalStorage : public INamedCollectionsStorage, protected WithContext
|
||||
{
|
||||
private:
|
||||
protected:
|
||||
std::string root_path;
|
||||
|
||||
public:
|
||||
@ -126,6 +131,11 @@ public:
|
||||
ReadBufferFromFile in(getPath(file_name));
|
||||
std::string data;
|
||||
readStringUntilEOF(data, in);
|
||||
return readHook(data);
|
||||
}
|
||||
|
||||
virtual std::string readHook(const std::string & data) const
|
||||
{
|
||||
return data;
|
||||
}
|
||||
|
||||
@ -142,8 +152,9 @@ public:
|
||||
fs::create_directories(root_path);
|
||||
|
||||
auto tmp_path = getPath(file_name + ".tmp");
|
||||
WriteBufferFromFile out(tmp_path, data.size(), O_WRONLY | O_CREAT | O_EXCL);
|
||||
writeString(data, out);
|
||||
auto write_data = writeHook(data);
|
||||
WriteBufferFromFile out(tmp_path, write_data.size(), O_WRONLY | O_CREAT | O_EXCL);
|
||||
writeString(write_data, out);
|
||||
|
||||
out.next();
|
||||
if (getContext()->getSettingsRef().fsync_metadata)
|
||||
@ -153,6 +164,11 @@ public:
|
||||
fs::rename(tmp_path, getPath(file_name));
|
||||
}
|
||||
|
||||
virtual std::string writeHook(const std::string & data) const
|
||||
{
|
||||
return data;
|
||||
}
|
||||
|
||||
void remove(const std::string & file_name) override
|
||||
{
|
||||
if (!removeIfExists(file_name))
|
||||
@ -168,7 +184,7 @@ public:
|
||||
return fs::remove(getPath(file_name));
|
||||
}
|
||||
|
||||
private:
|
||||
protected:
|
||||
std::string getPath(const std::string & file_name) const
|
||||
{
|
||||
const auto file_name_as_path = fs::path(file_name);
|
||||
@ -178,6 +194,7 @@ private:
|
||||
return fs::path(root_path) / file_name_as_path;
|
||||
}
|
||||
|
||||
private:
|
||||
/// Delete .tmp files. They could be left undeleted in case of
|
||||
/// some exception or abrupt server restart.
|
||||
void cleanup()
|
||||
@ -194,8 +211,7 @@ private:
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class NamedCollectionsMetadataStorage::ZooKeeperStorage : public INamedCollectionsStorage, private WithContext
|
||||
class NamedCollectionsMetadataStorage::ZooKeeperStorage : public INamedCollectionsStorage, protected WithContext
|
||||
{
|
||||
private:
|
||||
std::string root_path;
|
||||
@ -275,18 +291,25 @@ public:
|
||||
|
||||
std::string read(const std::string & file_name) const override
|
||||
{
|
||||
return getClient()->get(getPath(file_name));
|
||||
auto data = getClient()->get(getPath(file_name));
|
||||
return readHook(data);
|
||||
}
|
||||
|
||||
virtual std::string readHook(const std::string & data) const
|
||||
{
|
||||
return data;
|
||||
}
|
||||
|
||||
void write(const std::string & file_name, const std::string & data, bool replace) override
|
||||
{
|
||||
auto write_data = writeHook(data);
|
||||
if (replace)
|
||||
{
|
||||
getClient()->createOrUpdate(getPath(file_name), data, zkutil::CreateMode::Persistent);
|
||||
getClient()->createOrUpdate(getPath(file_name), write_data, zkutil::CreateMode::Persistent);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto code = getClient()->tryCreate(getPath(file_name), data, zkutil::CreateMode::Persistent);
|
||||
auto code = getClient()->tryCreate(getPath(file_name), write_data, zkutil::CreateMode::Persistent);
|
||||
|
||||
if (code == Coordination::Error::ZNODEEXISTS)
|
||||
{
|
||||
@ -298,6 +321,11 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
virtual std::string writeHook(const std::string & data) const
|
||||
{
|
||||
return data;
|
||||
}
|
||||
|
||||
void remove(const std::string & file_name) override
|
||||
{
|
||||
getClient()->remove(getPath(file_name));
|
||||
@ -334,6 +362,93 @@ private:
|
||||
}
|
||||
};
|
||||
|
||||
#if USE_SSL
|
||||
|
||||
template <typename BaseMetadataStorage>
|
||||
class NamedCollectionsMetadataStorageEncrypted : public BaseMetadataStorage
|
||||
{
|
||||
public:
|
||||
NamedCollectionsMetadataStorageEncrypted(ContextPtr context_, const std::string & path_)
|
||||
: BaseMetadataStorage(context_, path_)
|
||||
{
|
||||
const auto & config = BaseMetadataStorage::getContext()->getConfigRef();
|
||||
auto key_hex = config.getRawString("named_collections_storage.key_hex", "");
|
||||
try
|
||||
{
|
||||
key = boost::algorithm::unhex(key_hex);
|
||||
key_fingerprint = FileEncryption::calculateKeyFingerprint(key);
|
||||
}
|
||||
catch (const std::exception &)
|
||||
{
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot read key_hex, check for valid characters [0-9a-fA-F] and length");
|
||||
}
|
||||
|
||||
algorithm = FileEncryption::parseAlgorithmFromString(config.getString("named_collections_storage.algorithm", "aes_128_ctr"));
|
||||
}
|
||||
|
||||
std::string readHook(const std::string & data) const override
|
||||
{
|
||||
ReadBufferFromString in(data);
|
||||
Memory<> encrypted_buffer(data.length());
|
||||
|
||||
FileEncryption::Header header;
|
||||
try
|
||||
{
|
||||
header.read(in);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
e.addMessage("While reading the header of encrypted data");
|
||||
throw;
|
||||
}
|
||||
|
||||
size_t bytes_read = 0;
|
||||
while (bytes_read < encrypted_buffer.size() && !in.eof())
|
||||
{
|
||||
bytes_read += in.read(encrypted_buffer.data() + bytes_read, encrypted_buffer.size() - bytes_read);
|
||||
}
|
||||
|
||||
std::string decrypted_buffer;
|
||||
decrypted_buffer.resize(bytes_read);
|
||||
FileEncryption::Encryptor encryptor(header.algorithm, key, header.init_vector);
|
||||
encryptor.decrypt(encrypted_buffer.data(), bytes_read, decrypted_buffer.data());
|
||||
|
||||
return decrypted_buffer;
|
||||
}
|
||||
|
||||
std::string writeHook(const std::string & data) const override
|
||||
{
|
||||
FileEncryption::Header header{
|
||||
.algorithm = algorithm,
|
||||
.key_fingerprint = key_fingerprint,
|
||||
.init_vector = FileEncryption::InitVector::random()
|
||||
};
|
||||
|
||||
FileEncryption::Encryptor encryptor(header.algorithm, key, header.init_vector);
|
||||
WriteBufferFromOwnString out;
|
||||
header.write(out);
|
||||
encryptor.encrypt(data.data(), data.size(), out);
|
||||
return std::string(out.str());
|
||||
}
|
||||
|
||||
private:
|
||||
std::string key;
|
||||
UInt128 key_fingerprint;
|
||||
FileEncryption::Algorithm algorithm;
|
||||
};
|
||||
|
||||
class NamedCollectionsMetadataStorage::LocalStorageEncrypted : public NamedCollectionsMetadataStorageEncrypted<NamedCollectionsMetadataStorage::LocalStorage>
|
||||
{
|
||||
using NamedCollectionsMetadataStorageEncrypted<NamedCollectionsMetadataStorage::LocalStorage>::NamedCollectionsMetadataStorageEncrypted;
|
||||
};
|
||||
|
||||
class NamedCollectionsMetadataStorage::ZooKeeperStorageEncrypted : public NamedCollectionsMetadataStorageEncrypted<NamedCollectionsMetadataStorage::ZooKeeperStorage>
|
||||
{
|
||||
using NamedCollectionsMetadataStorageEncrypted<NamedCollectionsMetadataStorage::ZooKeeperStorage>::NamedCollectionsMetadataStorageEncrypted;
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
NamedCollectionsMetadataStorage::NamedCollectionsMetadataStorage(
|
||||
std::shared_ptr<INamedCollectionsStorage> storage_,
|
||||
ContextPtr context_)
|
||||
@ -495,7 +610,7 @@ std::unique_ptr<NamedCollectionsMetadataStorage> NamedCollectionsMetadataStorage
|
||||
const auto & config = context_->getConfigRef();
|
||||
const auto storage_type = config.getString(named_collections_storage_config_path + ".type", "local");
|
||||
|
||||
if (storage_type == "local")
|
||||
if (storage_type == "local" || storage_type == "local_encrypted")
|
||||
{
|
||||
const auto path = config.getString(
|
||||
named_collections_storage_config_path + ".path",
|
||||
@ -504,14 +619,36 @@ std::unique_ptr<NamedCollectionsMetadataStorage> NamedCollectionsMetadataStorage
|
||||
LOG_TRACE(getLogger("NamedCollectionsMetadataStorage"),
|
||||
"Using local storage for named collections at path: {}", path);
|
||||
|
||||
auto local_storage = std::make_unique<NamedCollectionsMetadataStorage::LocalStorage>(context_, path);
|
||||
std::unique_ptr<INamedCollectionsStorage> local_storage;
|
||||
if (storage_type == "local")
|
||||
local_storage = std::make_unique<NamedCollectionsMetadataStorage::LocalStorage>(context_, path);
|
||||
else if (storage_type == "local_encrypted")
|
||||
{
|
||||
#if USE_SSL
|
||||
local_storage = std::make_unique<NamedCollectionsMetadataStorage::LocalStorageEncrypted>(context_, path);
|
||||
#else
|
||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Named collections encryption requires building with SSL support");
|
||||
#endif
|
||||
}
|
||||
|
||||
return std::unique_ptr<NamedCollectionsMetadataStorage>(
|
||||
new NamedCollectionsMetadataStorage(std::move(local_storage), context_));
|
||||
}
|
||||
if (storage_type == "zookeeper" || storage_type == "keeper")
|
||||
if (storage_type == "zookeeper" || storage_type == "keeper" || storage_type == "zookeeper_encrypted" || storage_type == "keeper_encrypted")
|
||||
{
|
||||
const auto path = config.getString(named_collections_storage_config_path + ".path");
|
||||
auto zk_storage = std::make_unique<NamedCollectionsMetadataStorage::ZooKeeperStorage>(context_, path);
|
||||
|
||||
std::unique_ptr<INamedCollectionsStorage> zk_storage;
|
||||
if (!storage_type.ends_with("_encrypted"))
|
||||
zk_storage = std::make_unique<NamedCollectionsMetadataStorage::ZooKeeperStorage>(context_, path);
|
||||
else
|
||||
{
|
||||
#if USE_SSL
|
||||
zk_storage = std::make_unique<NamedCollectionsMetadataStorage::ZooKeeperStorageEncrypted>(context_, path);
|
||||
#else
|
||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Named collections encryption requires building with SSL support");
|
||||
#endif
|
||||
}
|
||||
|
||||
LOG_TRACE(getLogger("NamedCollectionsMetadataStorage"),
|
||||
"Using zookeeper storage for named collections at path: {}", path);
|
||||
|
||||
@ -35,7 +35,9 @@ public:
|
||||
private:
|
||||
class INamedCollectionsStorage;
|
||||
class LocalStorage;
|
||||
class LocalStorageEncrypted;
|
||||
class ZooKeeperStorage;
|
||||
class ZooKeeperStorageEncrypted;
|
||||
|
||||
std::shared_ptr<INamedCollectionsStorage> storage;
|
||||
|
||||
|
||||
@ -95,6 +95,27 @@ public:
|
||||
cells.erase(it);
|
||||
}
|
||||
|
||||
void remove(std::function<bool(const Key &, const MappedPtr &)> predicate) override
|
||||
{
|
||||
for (auto it = cells.begin(); it != cells.end();)
|
||||
{
|
||||
if (predicate(it->first, it->second.value))
|
||||
{
|
||||
auto & cell = it->second;
|
||||
|
||||
current_size_in_bytes -= cell.size;
|
||||
if (cell.is_protected)
|
||||
current_protected_size -= cell.size;
|
||||
|
||||
auto & queue = cell.is_protected ? protected_queue : probationary_queue;
|
||||
queue.erase(cell.queue_iterator);
|
||||
it = cells.erase(it);
|
||||
}
|
||||
else
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
MappedPtr get(const Key & key) override
|
||||
{
|
||||
auto it = cells.find(key);
|
||||
|
||||
@ -145,6 +145,23 @@ public:
|
||||
size_in_bytes -= sz;
|
||||
}
|
||||
|
||||
void remove(std::function<bool(const Key &, const MappedPtr &)> predicate) override
|
||||
{
|
||||
for (auto it = cache.begin(); it != cache.end();)
|
||||
{
|
||||
if (predicate(it->first, it->second))
|
||||
{
|
||||
size_t sz = weight_function(*it->second);
|
||||
if (it->first.user_id.has_value())
|
||||
Base::user_quotas->decreaseActual(*it->first.user_id, sz);
|
||||
it = cache.erase(it);
|
||||
size_in_bytes -= sz;
|
||||
}
|
||||
else
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
MappedPtr get(const Key & key) override
|
||||
{
|
||||
auto it = cache.find(key);
|
||||
|
||||
@ -1136,6 +1136,7 @@ class IColumn;
|
||||
M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \
|
||||
M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \
|
||||
M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \
|
||||
M(Bool, input_format_try_infer_variants, false, "Try to infer the Variant type in text formats when there is more than one possible type for column/array elements", 0) \
|
||||
M(Bool, type_json_skip_duplicated_paths, false, "When enabled, during parsing JSON object into JSON type duplicated paths will be ignored and only the first one will be inserted instead of an exception", 0) \
|
||||
M(UInt64, input_format_json_max_depth, 1000, "Maximum depth of a field in JSON. This is not a strict limit, it does not have to be applied precisely.", 0) \
|
||||
M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
|
||||
|
||||
@ -71,6 +71,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
|
||||
},
|
||||
{"24.9",
|
||||
{
|
||||
{"input_format_try_infer_variants", false, false, "Try to infer Variant type in text formats when there is more than one possible type for column/array elements"},
|
||||
}
|
||||
},
|
||||
{"24.8",
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <utility>
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <DataTypes/DataTypeInterval.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -212,6 +213,8 @@ static bool callOnIndexAndDataType(TypeIndex number, F && f, ExtraArgs && ... ar
|
||||
case TypeIndex::IPv4: return f(TypePair<DataTypeIPv4, T>(), std::forward<ExtraArgs>(args)...);
|
||||
case TypeIndex::IPv6: return f(TypePair<DataTypeIPv6, T>(), std::forward<ExtraArgs>(args)...);
|
||||
|
||||
case TypeIndex::Interval: return f(TypePair<DataTypeInterval, T>(), std::forward<ExtraArgs>(args)...);
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
@ -35,6 +35,7 @@ namespace ErrorCodes
|
||||
{
|
||||
extern const int UNEXPECTED_AST_STRUCTURE;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int CANNOT_COMPILE_REGEXP;
|
||||
}
|
||||
|
||||
DataTypeObject::DataTypeObject(
|
||||
@ -51,6 +52,17 @@ DataTypeObject::DataTypeObject(
|
||||
, max_dynamic_paths(max_dynamic_paths_)
|
||||
, max_dynamic_types(max_dynamic_types_)
|
||||
{
|
||||
/// Check if regular expressions are valid.
|
||||
for (const auto & regexp_str : path_regexps_to_skip)
|
||||
{
|
||||
re2::RE2::Options options;
|
||||
/// Don't log errors to stderr.
|
||||
options.set_log_errors(false);
|
||||
auto regexp = re2::RE2(regexp_str, options);
|
||||
if (!regexp.ok())
|
||||
throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, "Invalid regexp '{}': {}", regexp_str, regexp.error());
|
||||
}
|
||||
|
||||
for (const auto & [typed_path, type] : typed_paths)
|
||||
{
|
||||
for (const auto & path_to_skip : paths_to_skip)
|
||||
|
||||
@ -149,6 +149,8 @@ std::unique_ptr<IDataType::SubstreamData> IDataType::getSubcolumnData(
|
||||
|
||||
ISerialization::EnumerateStreamsSettings settings;
|
||||
settings.position_independent_encoding = false;
|
||||
/// Don't enumerate dynamic subcolumns, they are handled separately.
|
||||
settings.enumerate_dynamic_streams = false;
|
||||
data.serialization->enumerateStreams(settings, callback_with_data, data);
|
||||
|
||||
if (!res && data.type->hasDynamicSubcolumnsData())
|
||||
|
||||
@ -241,6 +241,10 @@ public:
|
||||
{
|
||||
SubstreamPath path;
|
||||
bool position_independent_encoding = true;
|
||||
/// If set to false, don't enumerate dynamic subcolumns
|
||||
/// (such as dynamic types in Dynamic column or dynamic paths in JSON column).
|
||||
/// It may be needed when dynamic subcolumns are processed separately.
|
||||
bool enumerate_dynamic_streams = true;
|
||||
};
|
||||
|
||||
virtual void enumerateStreams(
|
||||
|
||||
@ -64,7 +64,7 @@ void SerializationDynamic::enumerateStreams(
|
||||
const auto * deserialize_state = data.deserialize_state ? checkAndGetState<DeserializeBinaryBulkStateDynamic>(data.deserialize_state) : nullptr;
|
||||
|
||||
/// If column is nullptr and we don't have deserialize state yet, nothing to enumerate as we don't have any variants.
|
||||
if (!column_dynamic && !deserialize_state)
|
||||
if (!settings.enumerate_dynamic_streams || (!column_dynamic && !deserialize_state))
|
||||
return;
|
||||
|
||||
const auto & variant_type = column_dynamic ? column_dynamic->getVariantInfo().variant_type : checkAndGetState<DeserializeBinaryBulkStateDynamicStructure>(deserialize_state->structure_state)->variant_type;
|
||||
|
||||
@ -130,7 +130,7 @@ void SerializationObject::enumerateStreams(EnumerateStreamsSettings & settings,
|
||||
}
|
||||
|
||||
/// If column or deserialization state was provided, iterate over dynamic paths,
|
||||
if (column_object || structure_state)
|
||||
if (settings.enumerate_dynamic_streams && (column_object || structure_state))
|
||||
{
|
||||
/// Enumerate dynamic paths in sorted order for consistency.
|
||||
const auto * dynamic_paths = column_object ? &column_object->getDynamicPaths() : nullptr;
|
||||
|
||||
@ -228,6 +228,39 @@ void convertUInt64toInt64IfPossible(const DataTypes & types, TypeIndexSet & type
|
||||
}
|
||||
}
|
||||
|
||||
DataTypePtr findSmallestIntervalSuperType(const DataTypes &types, TypeIndexSet &types_set)
|
||||
{
|
||||
auto min_interval = IntervalKind::Kind::Year;
|
||||
DataTypePtr smallest_type;
|
||||
|
||||
bool is_higher_interval = false; // For Years, Quarters and Months
|
||||
|
||||
for (const auto &type : types)
|
||||
{
|
||||
if (const auto * interval_type = typeid_cast<const DataTypeInterval *>(type.get()))
|
||||
{
|
||||
auto current_interval = interval_type->getKind().kind;
|
||||
if (current_interval > IntervalKind::Kind::Week)
|
||||
is_higher_interval = true;
|
||||
if (current_interval < min_interval)
|
||||
{
|
||||
min_interval = current_interval;
|
||||
smallest_type = type;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (is_higher_interval && min_interval <= IntervalKind::Kind::Week)
|
||||
throw Exception(ErrorCodes::NO_COMMON_TYPE, "Cannot compare intervals {} and {} because the number of days in a month is not fixed", types[0]->getName(), types[1]->getName());
|
||||
|
||||
if (smallest_type)
|
||||
{
|
||||
types_set.clear();
|
||||
types_set.insert(smallest_type->getTypeId());
|
||||
}
|
||||
|
||||
return smallest_type;
|
||||
}
|
||||
}
|
||||
|
||||
template <LeastSupertypeOnError on_error>
|
||||
@ -652,6 +685,13 @@ DataTypePtr getLeastSupertype(const DataTypes & types)
|
||||
return numeric_type;
|
||||
}
|
||||
|
||||
/// For interval data types.
|
||||
{
|
||||
auto res = findSmallestIntervalSuperType(types, type_ids);
|
||||
if (res)
|
||||
return res;
|
||||
}
|
||||
|
||||
/// All other data types (UUID, AggregateFunction, Enum...) are compatible only if they are the same (checked in trivial cases).
|
||||
return throwOrReturn<on_error>(types, "", ErrorCodes::NO_COMMON_TYPE);
|
||||
}
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#pragma once
|
||||
#include <DataTypes/IDataType.h>
|
||||
#include <DataTypes/DataTypeInterval.h>
|
||||
#include <Common/IntervalKind.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -48,4 +50,7 @@ DataTypePtr getLeastSupertypeOrString(const TypeIndexSet & types);
|
||||
|
||||
DataTypePtr tryGetLeastSupertype(const TypeIndexSet & types);
|
||||
|
||||
/// A vector that shows the conversion rates to the next Interval type starting from NanoSecond
|
||||
static std::vector<int> interval_conversions = {1, 1000, 1000, 1000, 60, 60, 24, 7, 4, 3, 4};
|
||||
|
||||
}
|
||||
|
||||
@ -283,6 +283,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
|
||||
format_settings.max_parser_depth = context->getSettingsRef().max_parser_depth;
|
||||
format_settings.client_protocol_version = context->getClientProtocolVersion();
|
||||
format_settings.date_time_overflow_behavior = settings.date_time_overflow_behavior;
|
||||
format_settings.try_infer_variant = settings.input_format_try_infer_variants;
|
||||
|
||||
/// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context
|
||||
if (format_settings.schema.is_server)
|
||||
|
||||
@ -35,6 +35,7 @@ struct FormatSettings
|
||||
bool decimal_trailing_zeros = false;
|
||||
bool defaults_for_omitted_fields = true;
|
||||
bool is_writing_to_terminal = false;
|
||||
bool try_infer_variant = false;
|
||||
|
||||
bool seekable_read = true;
|
||||
UInt64 max_rows_to_read_for_schema_inference = 25000;
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeVariant.h>
|
||||
#include <DataTypes/DataTypeMap.h>
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
#include <DataTypes/DataTypeNothing.h>
|
||||
@ -306,6 +307,33 @@ namespace
|
||||
type_indexes.erase(TypeIndex::UInt64);
|
||||
}
|
||||
|
||||
/// if setting 'try_infer_variant' is true then we convert to type variant.
|
||||
void transformVariant(DataTypes & data_types, TypeIndexesSet & type_indexes)
|
||||
{
|
||||
if (checkIfTypesAreEqual(data_types))
|
||||
return;
|
||||
|
||||
DataTypes variant_types;
|
||||
for (const auto & type : data_types)
|
||||
{
|
||||
if (const auto * variant_type = typeid_cast<const DataTypeVariant *>(type.get()))
|
||||
{
|
||||
const auto & current_variants = variant_type->getVariants();
|
||||
variant_types.insert(variant_types.end(), current_variants.begin(), current_variants.end());
|
||||
}
|
||||
else
|
||||
{
|
||||
variant_types.push_back(type);
|
||||
}
|
||||
}
|
||||
|
||||
auto variant_type = std::make_shared<DataTypeVariant>(variant_types);
|
||||
|
||||
for (auto & type : data_types)
|
||||
type = variant_type;
|
||||
type_indexes = {TypeIndex::Variant};
|
||||
}
|
||||
|
||||
/// If we have only date/datetimes types (Date/DateTime/DateTime64), convert all of them to the common type,
|
||||
/// otherwise, convert all Date, DateTime and DateTime64 to String.
|
||||
void transformDatesAndDateTimes(DataTypes & data_types, TypeIndexesSet & type_indexes)
|
||||
@ -652,7 +680,11 @@ namespace
|
||||
transformDatesAndDateTimes(data_types, type_indexes);
|
||||
|
||||
if constexpr (!is_json)
|
||||
{
|
||||
if (settings.try_infer_variant)
|
||||
transformVariant(data_types, type_indexes);
|
||||
return;
|
||||
}
|
||||
|
||||
/// Check settings specific for JSON formats.
|
||||
|
||||
@ -670,6 +702,10 @@ namespace
|
||||
|
||||
if (settings.json.try_infer_objects_as_tuples)
|
||||
mergeJSONPaths(data_types, type_indexes, settings, json_info);
|
||||
|
||||
if (settings.try_infer_variant)
|
||||
transformVariant(data_types, type_indexes);
|
||||
|
||||
};
|
||||
|
||||
auto transform_complex_types = [&](DataTypes & data_types, TypeIndexesSet & type_indexes)
|
||||
@ -682,7 +718,11 @@ namespace
|
||||
transformNothingComplexTypes(data_types, type_indexes);
|
||||
|
||||
if constexpr (!is_json)
|
||||
{
|
||||
if (settings.try_infer_variant)
|
||||
transformVariant(data_types, type_indexes);
|
||||
return;
|
||||
}
|
||||
|
||||
/// Convert JSON tuples with same nested types to arrays.
|
||||
transformTuplesWithEqualNestedTypesToArrays(data_types, type_indexes);
|
||||
@ -695,6 +735,9 @@ namespace
|
||||
|
||||
if (json_info && json_info->allow_merging_named_tuples)
|
||||
mergeNamedTuples(data_types, type_indexes, settings, json_info);
|
||||
|
||||
if (settings.try_infer_variant)
|
||||
transformVariant(data_types, type_indexes);
|
||||
};
|
||||
|
||||
transformTypesRecursively(types, transform_simple_types, transform_complex_types);
|
||||
@ -861,7 +904,6 @@ namespace
|
||||
|
||||
if (checkIfTypesAreEqual(nested_types_copy))
|
||||
return std::make_shared<DataTypeArray>(nested_types_copy.back());
|
||||
|
||||
return std::make_shared<DataTypeTuple>(nested_types);
|
||||
}
|
||||
else
|
||||
@ -1456,6 +1498,15 @@ void transformFinalInferredJSONTypeIfNeededImpl(DataTypePtr & data_type, const F
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (const auto * variant_type = typeid_cast<const DataTypeVariant *>(data_type.get()))
|
||||
{
|
||||
auto nested_types = variant_type->getVariants();
|
||||
for (auto & nested_type : nested_types)
|
||||
transformFinalInferredJSONTypeIfNeededImpl(nested_type, settings, json_info, remain_nothing_types);
|
||||
data_type = std::make_shared<DataTypeVariant>(nested_types);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void transformFinalInferredJSONTypeIfNeeded(DataTypePtr & data_type, const FormatSettings & settings, JSONInferenceInfo * json_info)
|
||||
@ -1535,6 +1586,20 @@ DataTypePtr makeNullableRecursively(DataTypePtr type)
|
||||
return nested_type ? std::make_shared<DataTypeArray>(nested_type) : nullptr;
|
||||
}
|
||||
|
||||
if (which.isVariant())
|
||||
{
|
||||
const auto * variant_type = assert_cast<const DataTypeVariant *>(type.get());
|
||||
DataTypes nested_types;
|
||||
for (const auto & nested_type: variant_type->getVariants())
|
||||
{
|
||||
if (!nested_type->lowCardinality() && nested_type->haveSubtypes())
|
||||
nested_types.push_back(makeNullableRecursively(nested_type));
|
||||
else
|
||||
nested_types.push_back(nested_type);
|
||||
}
|
||||
return std::make_shared<DataTypeVariant>(nested_types);
|
||||
}
|
||||
|
||||
if (which.isTuple())
|
||||
{
|
||||
const auto * tuple_type = assert_cast<const DataTypeTuple *>(type.get());
|
||||
|
||||
@ -123,7 +123,7 @@ public:
|
||||
class Executor
|
||||
{
|
||||
public:
|
||||
static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, uint32_t parse_depth, uint32_t parse_backtracks, const ContextPtr & context)
|
||||
static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, uint32_t parse_depth, uint32_t parse_backtracks, bool function_json_value_return_type_allow_complex)
|
||||
{
|
||||
MutableColumnPtr to{result_type->createColumn()};
|
||||
to->reserve(input_rows_count);
|
||||
@ -191,7 +191,7 @@ public:
|
||||
{
|
||||
/// Instead of creating a new generator for each row, we can reuse the same one.
|
||||
generator_json_path.reinitialize();
|
||||
added_to_column = impl.insertResultToColumn(*to, document, generator_json_path, context);
|
||||
added_to_column = impl.insertResultToColumn(*to, document, generator_json_path, function_json_value_return_type_allow_complex);
|
||||
}
|
||||
if (!added_to_column)
|
||||
{
|
||||
@ -204,11 +204,18 @@ public:
|
||||
};
|
||||
|
||||
template <typename Name, template <typename, typename> typename Impl>
|
||||
class FunctionSQLJSON : public IFunction, WithConstContext
|
||||
class FunctionSQLJSON : public IFunction
|
||||
{
|
||||
public:
|
||||
static FunctionPtr create(ContextPtr context_) { return std::make_shared<FunctionSQLJSON>(context_); }
|
||||
explicit FunctionSQLJSON(ContextPtr context_) : WithConstContext(context_) { }
|
||||
explicit FunctionSQLJSON(ContextPtr context_)
|
||||
: max_parser_depth(context_->getSettingsRef().max_parser_depth),
|
||||
max_parser_backtracks(context_->getSettingsRef().max_parser_backtracks),
|
||||
allow_simdjson(context_->getSettingsRef().allow_simdjson),
|
||||
function_json_value_return_type_allow_complex(context_->getSettingsRef().function_json_value_return_type_allow_complex),
|
||||
function_json_value_return_type_allow_nullable(context_->getSettingsRef().function_json_value_return_type_allow_nullable)
|
||||
{
|
||||
}
|
||||
|
||||
static constexpr auto name = Name::name;
|
||||
String getName() const override { return Name::name; }
|
||||
@ -221,7 +228,7 @@ public:
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
return Impl<DummyJSONParser, DefaultJSONStringSerializer<DummyJSONParser::Element>>::getReturnType(
|
||||
Name::name, arguments, getContext());
|
||||
Name::name, arguments, function_json_value_return_type_allow_nullable);
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
|
||||
@ -231,19 +238,25 @@ public:
|
||||
/// 2. Create ASTPtr
|
||||
/// 3. Parser(Tokens, ASTPtr) -> complete AST
|
||||
/// 4. Execute functions: call getNextItem on generator and handle each item
|
||||
unsigned parse_depth = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_depth);
|
||||
unsigned parse_backtracks = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_backtracks);
|
||||
unsigned parse_depth = static_cast<unsigned>(max_parser_depth);
|
||||
unsigned parse_backtracks = static_cast<unsigned>(max_parser_backtracks);
|
||||
#if USE_SIMDJSON
|
||||
if (getContext()->getSettingsRef().allow_simdjson)
|
||||
if (allow_simdjson)
|
||||
return FunctionSQLJSONHelpers::Executor<
|
||||
Name,
|
||||
Impl<SimdJSONParser, JSONStringSerializer<SimdJSONParser::Element, SimdJSONElementFormatter>>,
|
||||
SimdJSONParser>::run(arguments, result_type, input_rows_count, parse_depth, parse_backtracks, getContext());
|
||||
SimdJSONParser>::run(arguments, result_type, input_rows_count, parse_depth, parse_backtracks, function_json_value_return_type_allow_complex);
|
||||
#endif
|
||||
return FunctionSQLJSONHelpers::
|
||||
Executor<Name, Impl<DummyJSONParser, DefaultJSONStringSerializer<DummyJSONParser::Element>>, DummyJSONParser>::run(
|
||||
arguments, result_type, input_rows_count, parse_depth, parse_backtracks, getContext());
|
||||
arguments, result_type, input_rows_count, parse_depth, parse_backtracks, function_json_value_return_type_allow_complex);
|
||||
}
|
||||
private:
|
||||
const size_t max_parser_depth;
|
||||
const size_t max_parser_backtracks;
|
||||
const bool allow_simdjson;
|
||||
const bool function_json_value_return_type_allow_complex;
|
||||
const bool function_json_value_return_type_allow_nullable;
|
||||
};
|
||||
|
||||
struct NameJSONExists
|
||||
@ -267,11 +280,11 @@ class JSONExistsImpl
|
||||
public:
|
||||
using Element = typename JSONParser::Element;
|
||||
|
||||
static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &, const ContextPtr &) { return std::make_shared<DataTypeUInt8>(); }
|
||||
static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &, bool) { return std::make_shared<DataTypeUInt8>(); }
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & root, GeneratorJSONPath<JSONParser> & generator_json_path, const ContextPtr &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & root, GeneratorJSONPath<JSONParser> & generator_json_path, bool)
|
||||
{
|
||||
Element current_element = root;
|
||||
VisitorStatus status;
|
||||
@ -305,9 +318,9 @@ class JSONValueImpl
|
||||
public:
|
||||
using Element = typename JSONParser::Element;
|
||||
|
||||
static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &, const ContextPtr & context)
|
||||
static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &, bool function_json_value_return_type_allow_nullable)
|
||||
{
|
||||
if (context->getSettingsRef().function_json_value_return_type_allow_nullable)
|
||||
if (function_json_value_return_type_allow_nullable)
|
||||
{
|
||||
DataTypePtr string_type = std::make_shared<DataTypeString>();
|
||||
return std::make_shared<DataTypeNullable>(string_type);
|
||||
@ -320,7 +333,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & root, GeneratorJSONPath<JSONParser> & generator_json_path, const ContextPtr & context)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & root, GeneratorJSONPath<JSONParser> & generator_json_path, bool function_json_value_return_type_allow_complex)
|
||||
{
|
||||
Element current_element = root;
|
||||
VisitorStatus status;
|
||||
@ -329,7 +342,7 @@ public:
|
||||
{
|
||||
if (status == VisitorStatus::Ok)
|
||||
{
|
||||
if (context->getSettingsRef().function_json_value_return_type_allow_complex)
|
||||
if (function_json_value_return_type_allow_complex)
|
||||
{
|
||||
break;
|
||||
}
|
||||
@ -383,11 +396,11 @@ class JSONQueryImpl
|
||||
public:
|
||||
using Element = typename JSONParser::Element;
|
||||
|
||||
static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &, const ContextPtr &) { return std::make_shared<DataTypeString>(); }
|
||||
static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &, bool) { return std::make_shared<DataTypeString>(); }
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & root, GeneratorJSONPath<JSONParser> & generator_json_path, const ContextPtr &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & root, GeneratorJSONPath<JSONParser> & generator_json_path, bool)
|
||||
{
|
||||
ColumnString & col_str = assert_cast<ColumnString &>(dest);
|
||||
|
||||
|
||||
@ -48,6 +48,7 @@
|
||||
#include <DataTypes/DataTypesBinaryEncoding.h>
|
||||
#include <DataTypes/ObjectUtils.h>
|
||||
#include <DataTypes/Serializations/SerializationDecimal.h>
|
||||
#include <DataTypes/getLeastSupertype.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <Functions/CastOverloadResolver.h>
|
||||
@ -1576,6 +1577,35 @@ struct ConvertImpl
|
||||
arguments, result_type, input_rows_count, additions);
|
||||
}
|
||||
}
|
||||
else if constexpr (std::is_same_v<FromDataType, DataTypeInterval> && std::is_same_v<ToDataType, DataTypeInterval>)
|
||||
{
|
||||
IntervalKind to = typeid_cast<const DataTypeInterval *>(result_type.get())->getKind();
|
||||
IntervalKind from = typeid_cast<const DataTypeInterval *>(arguments[0].type.get())->getKind();
|
||||
|
||||
if (from == to || arguments[0].column->empty())
|
||||
return arguments[0].column;
|
||||
|
||||
Int64 conversion_factor = 1;
|
||||
Int64 result_value;
|
||||
|
||||
int from_position = static_cast<int>(from.kind);
|
||||
int to_position = static_cast<int>(to.kind); /// Positions of each interval according to granularity map
|
||||
|
||||
if (from_position < to_position)
|
||||
{
|
||||
for (int i = from_position; i < to_position; ++i)
|
||||
conversion_factor *= interval_conversions[i];
|
||||
result_value = arguments[0].column->getInt(0) / conversion_factor;
|
||||
}
|
||||
else
|
||||
{
|
||||
for (int i = from_position; i > to_position; --i)
|
||||
conversion_factor *= interval_conversions[i];
|
||||
result_value = arguments[0].column->getInt(0) * conversion_factor;
|
||||
}
|
||||
|
||||
return ColumnConst::create(ColumnInt64::create(1, result_value), input_rows_count);
|
||||
}
|
||||
else
|
||||
{
|
||||
using FromFieldType = typename FromDataType::FieldType;
|
||||
@ -2184,7 +2214,7 @@ private:
|
||||
const DataTypePtr from_type = removeNullable(arguments[0].type);
|
||||
ColumnPtr result_column;
|
||||
|
||||
[[maybe_unused]] FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior = default_date_time_overflow_behavior;
|
||||
FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior = default_date_time_overflow_behavior;
|
||||
|
||||
if (context)
|
||||
date_time_overflow_behavior = context->getSettingsRef().date_time_overflow_behavior.value;
|
||||
@ -2337,6 +2367,10 @@ private:
|
||||
else
|
||||
done = callOnIndexAndDataType<ToDataType>(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag);
|
||||
}
|
||||
|
||||
if constexpr (std::is_same_v<ToDataType, DataTypeInterval>)
|
||||
if (WhichDataType(from_type).isInterval())
|
||||
done = callOnIndexAndDataType<ToDataType>(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag);
|
||||
}
|
||||
|
||||
if (!done)
|
||||
|
||||
@ -77,64 +77,70 @@ namespace impl
|
||||
ColumnPtr key0;
|
||||
ColumnPtr key1;
|
||||
bool is_const;
|
||||
const ColumnArray::Offsets * offsets{};
|
||||
const ColumnArray::Offsets * offsets = nullptr;
|
||||
|
||||
size_t size() const
|
||||
{
|
||||
assert(key0 && key1);
|
||||
assert(key0->size() == key1->size());
|
||||
assert(offsets == nullptr || offsets->size() == key0->size());
|
||||
if (offsets != nullptr)
|
||||
if (offsets != nullptr && !offsets->empty())
|
||||
return offsets->back();
|
||||
return key0->size();
|
||||
}
|
||||
|
||||
SipHashKey getKey(size_t i) const
|
||||
{
|
||||
if (is_const)
|
||||
i = 0;
|
||||
assert(key0->size() == key1->size());
|
||||
if (offsets != nullptr)
|
||||
{
|
||||
const auto *const begin = offsets->begin();
|
||||
const auto * const begin = offsets->begin();
|
||||
const auto * upper = std::upper_bound(begin, offsets->end(), i);
|
||||
if (upper == offsets->end())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "offset {} not found in function SipHashKeyColumns::getKey", i);
|
||||
if (upper != offsets->end())
|
||||
i = upper - begin;
|
||||
}
|
||||
const auto & key0data = assert_cast<const ColumnUInt64 &>(*key0).getData();
|
||||
const auto & key1data = assert_cast<const ColumnUInt64 &>(*key1).getData();
|
||||
assert(key0->size() > i);
|
||||
return {key0data[i], key1data[i]};
|
||||
}
|
||||
};
|
||||
|
||||
static SipHashKeyColumns parseSipHashKeyColumns(const ColumnWithTypeAndName & key)
|
||||
{
|
||||
const ColumnTuple * tuple = nullptr;
|
||||
const auto * column = key.column.get();
|
||||
bool is_const = false;
|
||||
if (isColumnConst(*column))
|
||||
const auto * col_key = key.column.get();
|
||||
|
||||
bool is_const;
|
||||
const ColumnTuple * col_key_tuple;
|
||||
if (isColumnConst(*col_key))
|
||||
{
|
||||
is_const = true;
|
||||
tuple = checkAndGetColumnConstData<ColumnTuple>(column);
|
||||
col_key_tuple = checkAndGetColumnConstData<ColumnTuple>(col_key);
|
||||
}
|
||||
else
|
||||
tuple = checkAndGetColumn<ColumnTuple>(column);
|
||||
if (!tuple)
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "key must be a tuple");
|
||||
if (tuple->tupleSize() != 2)
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "wrong tuple size: key must be a tuple of 2 UInt64");
|
||||
{
|
||||
is_const = false;
|
||||
col_key_tuple = checkAndGetColumn<ColumnTuple>(col_key);
|
||||
}
|
||||
|
||||
SipHashKeyColumns ret{tuple->getColumnPtr(0), tuple->getColumnPtr(1), is_const};
|
||||
assert(ret.key0);
|
||||
if (!checkColumn<ColumnUInt64>(*ret.key0))
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "first element of the key tuple is not UInt64");
|
||||
assert(ret.key1);
|
||||
if (!checkColumn<ColumnUInt64>(*ret.key1))
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "second element of the key tuple is not UInt64");
|
||||
if (!col_key_tuple || col_key_tuple->tupleSize() != 2)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The key must be of type Tuple(UInt64, UInt64)");
|
||||
|
||||
if (ret.size() == 1)
|
||||
ret.is_const = true;
|
||||
SipHashKeyColumns result{.key0 = col_key_tuple->getColumnPtr(0), .key1 = col_key_tuple->getColumnPtr(1), .is_const = is_const};
|
||||
|
||||
return ret;
|
||||
assert(result.key0);
|
||||
assert(result.key1);
|
||||
|
||||
if (!checkColumn<ColumnUInt64>(*result.key0))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The 1st element of the key tuple is not of type UInt64");
|
||||
if (!checkColumn<ColumnUInt64>(*result.key1))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The 2nd element of the key tuple is not of type UInt64");
|
||||
|
||||
if (result.size() == 1)
|
||||
result.is_const = true;
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -6,7 +6,6 @@
|
||||
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Functions/LowerUpperImpl.h>
|
||||
#include <base/find_symbols.h>
|
||||
#include <unicode/unistr.h>
|
||||
#include <Common/StringUtils.h>
|
||||
|
||||
@ -43,7 +42,7 @@ struct LowerUpperUTF8Impl
|
||||
|
||||
String output;
|
||||
size_t curr_offset = 0;
|
||||
for (size_t i = 0; i < offsets.size(); ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
const auto * data_start = reinterpret_cast<const char *>(&data[offsets[i - 1]]);
|
||||
size_t size = offsets[i] - offsets[i - 1];
|
||||
@ -57,13 +56,15 @@ struct LowerUpperUTF8Impl
|
||||
output.clear();
|
||||
input.toUTF8String(output);
|
||||
|
||||
/// For valid UTF-8 input strings, ICU sometimes produces output with extra '\0's at the end. Only the data before the first
|
||||
/// '\0' is valid. It the input is not valid UTF-8, then the behavior of lower/upperUTF8 is undefined by definition. In this
|
||||
/// For valid UTF-8 input strings, ICU sometimes produces output with an extra '\0 at the end. Only the data before that
|
||||
/// '\0' is valid. If the input is not valid UTF-8, then the behavior of lower/upperUTF8 is undefined by definition. In this
|
||||
/// case, the behavior is also reasonable.
|
||||
const char * res_end = find_last_not_symbols_or_null<'\0'>(output.data(), output.data() + output.size());
|
||||
size_t valid_size = res_end ? res_end - output.data() + 1 : 0;
|
||||
size_t valid_size = output.size();
|
||||
if (!output.empty() && output.back() == '\0')
|
||||
--valid_size;
|
||||
|
||||
res_data.resize(curr_offset + valid_size + 1);
|
||||
|
||||
memcpy(&res_data[curr_offset], output.data(), valid_size);
|
||||
res_data[curr_offset + valid_size] = 0;
|
||||
|
||||
|
||||
718
src/Functions/overlay.cpp
Normal file
718
src/Functions/overlay.cpp
Normal file
@ -0,0 +1,718 @@
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/GatherUtils/Sources.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Common/StringUtils.h>
|
||||
#include <Common/UTF8Helpers.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// If 'is_utf8' - measure offset and length in code points instead of bytes.
|
||||
/// Syntax:
|
||||
/// - overlay(input, replace, offset[, length])
|
||||
/// - overlayUTF8(input, replace, offset[, length]) - measure offset and length in code points instead of bytes
|
||||
template <bool is_utf8>
|
||||
class FunctionOverlay : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = is_utf8 ? "overlayUTF8" : "overlay";
|
||||
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionOverlay>(); }
|
||||
String getName() const override { return name; }
|
||||
bool isVariadic() const override { return true; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors mandatory_args{
|
||||
{"input", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
|
||||
{"replace", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
|
||||
{"offset", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), nullptr, "(U)Int8/16/32/64"},
|
||||
};
|
||||
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"length", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), nullptr, "(U)Int8/16/32/64"},
|
||||
};
|
||||
|
||||
validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
|
||||
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (input_rows_count == 0)
|
||||
return ColumnString::create();
|
||||
|
||||
bool has_four_args = (arguments.size() == 4);
|
||||
|
||||
ColumnPtr col_input = arguments[0].column;
|
||||
const auto * col_input_const = checkAndGetColumn<ColumnConst>(col_input.get());
|
||||
const auto * col_input_string = checkAndGetColumn<ColumnString>(col_input.get());
|
||||
bool input_is_const = (col_input_const != nullptr);
|
||||
|
||||
ColumnPtr col_replace = arguments[1].column;
|
||||
const auto * col_replace_const = checkAndGetColumn<ColumnConst>(col_replace.get());
|
||||
const auto * col_replace_string = checkAndGetColumn<ColumnString>(col_replace.get());
|
||||
bool replace_is_const = (col_replace_const != nullptr);
|
||||
|
||||
ColumnPtr col_offset = arguments[2].column;
|
||||
const ColumnConst * col_offset_const = checkAndGetColumn<ColumnConst>(col_offset.get());
|
||||
bool offset_is_const = false;
|
||||
Int64 offset = -1;
|
||||
if (col_offset_const)
|
||||
{
|
||||
offset = col_offset_const->getInt(0);
|
||||
offset_is_const = true;
|
||||
}
|
||||
|
||||
ColumnPtr col_length = has_four_args ? arguments[3].column : nullptr;
|
||||
const ColumnConst * col_length_const = has_four_args ? checkAndGetColumn<ColumnConst>(col_length.get()) : nullptr;
|
||||
bool length_is_const = false;
|
||||
Int64 length = -1;
|
||||
if (col_length_const)
|
||||
{
|
||||
length = col_length_const->getInt(0);
|
||||
length_is_const = true;
|
||||
}
|
||||
|
||||
auto res_col = ColumnString::create();
|
||||
auto & res_data = res_col->getChars();
|
||||
auto & res_offsets = res_col->getOffsets();
|
||||
|
||||
res_offsets.resize_exact(input_rows_count);
|
||||
if (col_input_const)
|
||||
{
|
||||
StringRef input = col_input_const->getDataAt(0);
|
||||
res_data.reserve((input.size + 1) * input_rows_count);
|
||||
}
|
||||
else
|
||||
{
|
||||
res_data.reserve(col_input_string->getChars().size());
|
||||
}
|
||||
|
||||
|
||||
#define OVERLAY_EXECUTE_CASE(HAS_FOUR_ARGS, OFFSET_IS_CONST, LENGTH_IS_CONST) \
|
||||
if (input_is_const && replace_is_const) \
|
||||
constantConstant<HAS_FOUR_ARGS, OFFSET_IS_CONST, LENGTH_IS_CONST>( \
|
||||
input_rows_count, \
|
||||
col_input_const->getDataAt(0), \
|
||||
col_replace_const->getDataAt(0), \
|
||||
col_offset, \
|
||||
col_length, \
|
||||
offset, \
|
||||
length, \
|
||||
res_data, \
|
||||
res_offsets); \
|
||||
else if (input_is_const && !replace_is_const) \
|
||||
constantVector<HAS_FOUR_ARGS, OFFSET_IS_CONST, LENGTH_IS_CONST>( \
|
||||
input_rows_count, \
|
||||
col_input_const->getDataAt(0), \
|
||||
col_replace_string->getChars(), \
|
||||
col_replace_string->getOffsets(), \
|
||||
col_offset, \
|
||||
col_length, \
|
||||
offset, \
|
||||
length, \
|
||||
res_data, \
|
||||
res_offsets); \
|
||||
else if (!input_is_const && replace_is_const) \
|
||||
vectorConstant<HAS_FOUR_ARGS, OFFSET_IS_CONST, LENGTH_IS_CONST>( \
|
||||
input_rows_count, \
|
||||
col_input_string->getChars(), \
|
||||
col_input_string->getOffsets(), \
|
||||
col_replace_const->getDataAt(0), \
|
||||
col_offset, \
|
||||
col_length, \
|
||||
offset, \
|
||||
length, \
|
||||
res_data, \
|
||||
res_offsets); \
|
||||
else \
|
||||
vectorVector<HAS_FOUR_ARGS, OFFSET_IS_CONST, LENGTH_IS_CONST>( \
|
||||
input_rows_count, \
|
||||
col_input_string->getChars(), \
|
||||
col_input_string->getOffsets(), \
|
||||
col_replace_string->getChars(), \
|
||||
col_replace_string->getOffsets(), \
|
||||
col_offset, \
|
||||
col_length, \
|
||||
offset, \
|
||||
length, \
|
||||
res_data, \
|
||||
res_offsets);
|
||||
|
||||
if (!has_four_args)
|
||||
{
|
||||
if (offset_is_const)
|
||||
{
|
||||
OVERLAY_EXECUTE_CASE(false, true, false)
|
||||
}
|
||||
else
|
||||
{
|
||||
OVERLAY_EXECUTE_CASE(false, false, false)
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (offset_is_const && length_is_const)
|
||||
{
|
||||
OVERLAY_EXECUTE_CASE(true, true, true)
|
||||
}
|
||||
else if (offset_is_const && !length_is_const)
|
||||
{
|
||||
OVERLAY_EXECUTE_CASE(true, true, false)
|
||||
}
|
||||
else if (!offset_is_const && length_is_const)
|
||||
{
|
||||
OVERLAY_EXECUTE_CASE(true, false, true)
|
||||
}
|
||||
else
|
||||
{
|
||||
OVERLAY_EXECUTE_CASE(true, false, false)
|
||||
}
|
||||
}
|
||||
#undef OVERLAY_EXECUTE_CASE
|
||||
|
||||
return res_col;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
/// input offset is 1-based, maybe negative
|
||||
/// output result is 0-based valid offset, within [0, input_size]
|
||||
static size_t getValidOffset(Int64 offset, size_t input_size)
|
||||
{
|
||||
if (offset > 0)
|
||||
{
|
||||
if (static_cast<size_t>(offset) > input_size + 1)
|
||||
return input_size;
|
||||
else
|
||||
return offset - 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (input_size < -static_cast<size_t>(offset))
|
||||
return 0;
|
||||
else
|
||||
return input_size + offset;
|
||||
}
|
||||
}
|
||||
|
||||
/// get character count of a slice [data, data+bytes)
|
||||
static size_t getSliceSize(const UInt8 * data, size_t bytes)
|
||||
{
|
||||
if constexpr (is_utf8)
|
||||
return UTF8::countCodePoints(data, bytes);
|
||||
else
|
||||
return bytes;
|
||||
}
|
||||
|
||||
template <bool has_four_args, bool offset_is_const, bool length_is_const>
|
||||
void constantConstant(
|
||||
size_t rows,
|
||||
const StringRef & input,
|
||||
const StringRef & replace,
|
||||
const ColumnPtr & column_offset,
|
||||
const ColumnPtr & column_length,
|
||||
Int64 const_offset,
|
||||
Int64 const_length,
|
||||
ColumnString::Chars & res_data,
|
||||
ColumnString::Offsets & res_offsets) const
|
||||
{
|
||||
if (has_four_args && length_is_const && const_length < 0)
|
||||
{
|
||||
constantConstant<true, offset_is_const, false>(
|
||||
rows, input, replace, column_offset, column_length, const_offset, -1, res_data, res_offsets);
|
||||
return;
|
||||
}
|
||||
|
||||
size_t input_size = getSliceSize(reinterpret_cast<const UInt8 *>(input.data), input.size);
|
||||
size_t valid_offset = 0; // start from 0, not negative
|
||||
if constexpr (offset_is_const)
|
||||
valid_offset = getValidOffset(const_offset, input_size);
|
||||
|
||||
size_t replace_size = getSliceSize(reinterpret_cast<const UInt8 *>(replace.data), replace.size);
|
||||
size_t valid_length = 0; // not negative
|
||||
if constexpr (has_four_args && length_is_const)
|
||||
{
|
||||
assert(const_length >= 0);
|
||||
valid_length = const_length;
|
||||
}
|
||||
else if constexpr (!has_four_args)
|
||||
{
|
||||
valid_length = replace_size;
|
||||
}
|
||||
|
||||
Int64 offset = 0; // start from 1, maybe negative
|
||||
Int64 length = 0; // maybe negative
|
||||
const UInt8 * input_begin = reinterpret_cast<const UInt8 *>(input.data);
|
||||
const UInt8 * input_end = reinterpret_cast<const UInt8 *>(input.data + input.size);
|
||||
size_t res_offset = 0;
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
if constexpr (!offset_is_const)
|
||||
{
|
||||
offset = column_offset->getInt(i);
|
||||
valid_offset = getValidOffset(offset, input_size);
|
||||
}
|
||||
|
||||
if constexpr (has_four_args && !length_is_const)
|
||||
{
|
||||
length = column_length->getInt(i);
|
||||
valid_length = length >= 0 ? length : replace_size;
|
||||
}
|
||||
|
||||
size_t prefix_size = valid_offset;
|
||||
size_t suffix_size = (prefix_size + valid_length > input_size) ? 0 : (input_size - prefix_size - valid_length);
|
||||
|
||||
if constexpr (!is_utf8)
|
||||
{
|
||||
size_t new_res_size = res_data.size() + prefix_size + replace_size + suffix_size + 1; /// +1 for zero terminator
|
||||
res_data.resize(new_res_size);
|
||||
|
||||
/// copy prefix before replaced region
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input.data, prefix_size);
|
||||
res_offset += prefix_size;
|
||||
|
||||
/// copy replace
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], replace.data, replace_size);
|
||||
res_offset += replace_size;
|
||||
|
||||
/// copy suffix after replaced region. It is not necessary to copy if suffix_size is zero.
|
||||
if (suffix_size)
|
||||
{
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input.data + prefix_size + valid_length, suffix_size);
|
||||
res_offset += suffix_size;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto * prefix_end = GatherUtils::UTF8StringSource::skipCodePointsForward(input_begin, prefix_size, input_end);
|
||||
size_t prefix_bytes = prefix_end > input_end ? input.size : prefix_end - input_begin;
|
||||
|
||||
const auto * suffix_begin = GatherUtils::UTF8StringSource::skipCodePointsBackward(input_end, suffix_size, input_begin);
|
||||
size_t suffix_bytes = input_end - suffix_begin;
|
||||
|
||||
size_t new_res_size = res_data.size() + prefix_bytes + replace.size + suffix_bytes + 1; /// +1 for zero terminator
|
||||
res_data.resize(new_res_size);
|
||||
|
||||
/// copy prefix before replaced region
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input_begin, prefix_bytes);
|
||||
res_offset += prefix_bytes;
|
||||
|
||||
/// copy replace
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], replace.data, replace.size);
|
||||
res_offset += replace.size;
|
||||
|
||||
/// copy suffix after replaced region. It is not necessary to copy if suffix_bytes is zero.
|
||||
if (suffix_bytes)
|
||||
{
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], suffix_begin, suffix_bytes);
|
||||
res_offset += suffix_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/// add zero terminator
|
||||
res_data[res_offset] = 0;
|
||||
++res_offset;
|
||||
|
||||
res_offsets[i] = res_offset;
|
||||
}
|
||||
}
|
||||
|
||||
template <bool has_four_args, bool offset_is_const, bool length_is_const>
|
||||
void vectorConstant(
|
||||
size_t rows,
|
||||
const ColumnString::Chars & input_data,
|
||||
const ColumnString::Offsets & input_offsets,
|
||||
const StringRef & replace,
|
||||
const ColumnPtr & column_offset,
|
||||
const ColumnPtr & column_length,
|
||||
Int64 const_offset,
|
||||
Int64 const_length,
|
||||
ColumnString::Chars & res_data,
|
||||
ColumnString::Offsets & res_offsets) const
|
||||
{
|
||||
if (has_four_args && length_is_const && const_length < 0)
|
||||
{
|
||||
vectorConstant<true, offset_is_const, false>(
|
||||
rows, input_data, input_offsets, replace, column_offset, column_length, const_offset, -1, res_data, res_offsets);
|
||||
return;
|
||||
}
|
||||
|
||||
size_t replace_size = getSliceSize(reinterpret_cast<const UInt8 *>(replace.data), replace.size);
|
||||
Int64 length = 0; // maybe negative
|
||||
size_t valid_length = 0; // not negative
|
||||
if constexpr (has_four_args && length_is_const)
|
||||
{
|
||||
assert(const_length >= 0);
|
||||
valid_length = const_length;
|
||||
}
|
||||
else if constexpr (!has_four_args)
|
||||
{
|
||||
valid_length = replace_size;
|
||||
}
|
||||
|
||||
Int64 offset = 0; // start from 1, maybe negative
|
||||
size_t valid_offset = 0; // start from 0, not negative
|
||||
size_t res_offset = 0;
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
size_t input_offset = input_offsets[i - 1];
|
||||
size_t input_bytes = input_offsets[i] - input_offsets[i - 1] - 1;
|
||||
size_t input_size = getSliceSize(&input_data[input_offset], input_bytes);
|
||||
|
||||
if constexpr (offset_is_const)
|
||||
{
|
||||
valid_offset = getValidOffset(const_offset, input_size);
|
||||
}
|
||||
else
|
||||
{
|
||||
offset = column_offset->getInt(i);
|
||||
valid_offset = getValidOffset(offset, input_size);
|
||||
}
|
||||
|
||||
if constexpr (has_four_args && !length_is_const)
|
||||
{
|
||||
length = column_length->getInt(i);
|
||||
valid_length = length >= 0 ? length : replace_size;
|
||||
}
|
||||
|
||||
size_t prefix_size = valid_offset;
|
||||
size_t suffix_size = (prefix_size + valid_length > input_size) ? 0 : (input_size - prefix_size - valid_length);
|
||||
|
||||
if constexpr (!is_utf8)
|
||||
{
|
||||
size_t new_res_size = res_data.size() + prefix_size + replace_size + suffix_size + 1; /// +1 for zero terminator
|
||||
res_data.resize(new_res_size);
|
||||
|
||||
/// copy prefix before replaced region
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &input_data[input_offset], prefix_size);
|
||||
res_offset += prefix_size;
|
||||
|
||||
/// copy replace
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], replace.data, replace_size);
|
||||
res_offset += replace_size;
|
||||
|
||||
/// copy suffix after replaced region. It is not necessary to copy if suffix_size is zero.
|
||||
if (suffix_size)
|
||||
{
|
||||
memcpySmallAllowReadWriteOverflow15(
|
||||
&res_data[res_offset], &input_data[input_offset + prefix_size + valid_length], suffix_size);
|
||||
res_offset += suffix_size;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto * input_begin = &input_data[input_offset];
|
||||
const auto * input_end = &input_data[input_offset + input_bytes];
|
||||
const auto * prefix_end = GatherUtils::UTF8StringSource::skipCodePointsForward(input_begin, prefix_size, input_end);
|
||||
size_t prefix_bytes = prefix_end > input_end ? input_bytes : prefix_end - input_begin;
|
||||
const auto * suffix_begin = GatherUtils::UTF8StringSource::skipCodePointsBackward(input_end, suffix_size, input_begin);
|
||||
size_t suffix_bytes = input_end - suffix_begin;
|
||||
|
||||
size_t new_res_size = res_data.size() + prefix_bytes + replace.size + suffix_bytes + 1; /// +1 for zero terminator
|
||||
res_data.resize(new_res_size);
|
||||
|
||||
/// copy prefix before replaced region
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &input_data[input_offset], prefix_bytes);
|
||||
res_offset += prefix_bytes;
|
||||
|
||||
/// copy replace
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], replace.data, replace.size);
|
||||
res_offset += replace.size;
|
||||
|
||||
/// copy suffix after replaced region. It is not necessary to copy if suffix_bytes is zero.
|
||||
if (suffix_bytes)
|
||||
{
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], suffix_begin, suffix_bytes);
|
||||
res_offset += suffix_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/// add zero terminator
|
||||
res_data[res_offset] = 0;
|
||||
++res_offset;
|
||||
|
||||
res_offsets[i] = res_offset;
|
||||
}
|
||||
}
|
||||
|
||||
template <bool has_four_args, bool offset_is_const, bool length_is_const>
|
||||
void constantVector(
|
||||
size_t rows,
|
||||
const StringRef & input,
|
||||
const ColumnString::Chars & replace_data,
|
||||
const ColumnString::Offsets & replace_offsets,
|
||||
const ColumnPtr & column_offset,
|
||||
const ColumnPtr & column_length,
|
||||
Int64 const_offset,
|
||||
Int64 const_length,
|
||||
ColumnString::Chars & res_data,
|
||||
ColumnString::Offsets & res_offsets) const
|
||||
{
|
||||
if (has_four_args && length_is_const && const_length < 0)
|
||||
{
|
||||
constantVector<true, offset_is_const, false>(
|
||||
rows, input, replace_data, replace_offsets, column_offset, column_length, const_offset, -1, res_data, res_offsets);
|
||||
return;
|
||||
}
|
||||
|
||||
size_t input_size = getSliceSize(reinterpret_cast<const UInt8 *>(input.data), input.size);
|
||||
size_t valid_offset = 0; // start from 0, not negative
|
||||
if constexpr (offset_is_const)
|
||||
valid_offset = getValidOffset(const_offset, input_size);
|
||||
|
||||
Int64 length = 0; // maybe negative
|
||||
size_t valid_length = 0; // not negative
|
||||
if constexpr (has_four_args && length_is_const)
|
||||
{
|
||||
assert(const_length >= 0);
|
||||
valid_length = const_length;
|
||||
}
|
||||
|
||||
const auto * input_begin = reinterpret_cast<const UInt8 *>(input.data);
|
||||
const auto * input_end = reinterpret_cast<const UInt8 *>(input.data + input.size);
|
||||
Int64 offset = 0; // start from 1, maybe negative
|
||||
size_t res_offset = 0;
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
size_t replace_offset = replace_offsets[i - 1];
|
||||
size_t replace_bytes = replace_offsets[i] - replace_offsets[i - 1] - 1;
|
||||
size_t replace_size = getSliceSize(&replace_data[replace_offset], replace_bytes);
|
||||
|
||||
if constexpr (!offset_is_const)
|
||||
{
|
||||
offset = column_offset->getInt(i);
|
||||
valid_offset = getValidOffset(offset, input_size);
|
||||
}
|
||||
|
||||
if constexpr (!has_four_args)
|
||||
{
|
||||
valid_length = replace_size;
|
||||
}
|
||||
else if constexpr (!length_is_const)
|
||||
{
|
||||
length = column_length->getInt(i);
|
||||
valid_length = length >= 0 ? length : replace_size;
|
||||
}
|
||||
|
||||
size_t prefix_size = valid_offset;
|
||||
size_t suffix_size = (prefix_size + valid_length > input_size) ? 0 : (input_size - prefix_size - valid_length);
|
||||
|
||||
if constexpr (!is_utf8)
|
||||
{
|
||||
size_t new_res_size = res_data.size() + prefix_size + replace_size + suffix_size + 1; /// +1 for zero terminator
|
||||
res_data.resize(new_res_size);
|
||||
|
||||
/// copy prefix before replaced region
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input.data, prefix_size);
|
||||
res_offset += prefix_size;
|
||||
|
||||
/// copy replace
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &replace_data[replace_offset], replace_size);
|
||||
res_offset += replace_size;
|
||||
|
||||
/// copy suffix after replaced region. It is not necessary to copy if suffix_size is zero.
|
||||
if (suffix_size)
|
||||
{
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input.data + prefix_size + valid_length, suffix_size);
|
||||
res_offset += suffix_size;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto * prefix_end = GatherUtils::UTF8StringSource::skipCodePointsForward(input_begin, prefix_size, input_end);
|
||||
size_t prefix_bytes = prefix_end > input_end ? input.size : prefix_end - input_begin;
|
||||
const auto * suffix_begin = GatherUtils::UTF8StringSource::skipCodePointsBackward(input_end, suffix_size, input_begin);
|
||||
size_t suffix_bytes = input_end - suffix_begin;
|
||||
size_t new_res_size = res_data.size() + prefix_bytes + replace_bytes + suffix_bytes + 1; /// +1 for zero terminator
|
||||
res_data.resize(new_res_size);
|
||||
|
||||
/// copy prefix before replaced region
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input_begin, prefix_bytes);
|
||||
res_offset += prefix_bytes;
|
||||
|
||||
/// copy replace
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &replace_data[replace_offset], replace_bytes);
|
||||
res_offset += replace_bytes;
|
||||
|
||||
/// copy suffix after replaced region. It is not necessary to copy if suffix_bytes is zero
|
||||
if (suffix_bytes)
|
||||
{
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], suffix_begin, suffix_bytes);
|
||||
res_offset += suffix_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/// add zero terminator
|
||||
res_data[res_offset] = 0;
|
||||
++res_offset;
|
||||
|
||||
res_offsets[i] = res_offset;
|
||||
}
|
||||
}
|
||||
|
||||
template <bool has_four_args, bool offset_is_const, bool length_is_const>
|
||||
void vectorVector(
|
||||
size_t rows,
|
||||
const ColumnString::Chars & input_data,
|
||||
const ColumnString::Offsets & input_offsets,
|
||||
const ColumnString::Chars & replace_data,
|
||||
const ColumnString::Offsets & replace_offsets,
|
||||
const ColumnPtr & column_offset,
|
||||
const ColumnPtr & column_length,
|
||||
Int64 const_offset,
|
||||
Int64 const_length,
|
||||
ColumnString::Chars & res_data,
|
||||
ColumnString::Offsets & res_offsets) const
|
||||
{
|
||||
if (has_four_args && length_is_const && const_length < 0)
|
||||
{
|
||||
vectorVector<true, offset_is_const, false>(
|
||||
rows,
|
||||
input_data,
|
||||
input_offsets,
|
||||
replace_data,
|
||||
replace_offsets,
|
||||
column_offset,
|
||||
column_length,
|
||||
const_offset,
|
||||
-1,
|
||||
res_data,
|
||||
res_offsets);
|
||||
return;
|
||||
}
|
||||
|
||||
Int64 length = 0; // maybe negative
|
||||
size_t valid_length = 0; // not negative
|
||||
if constexpr (has_four_args && length_is_const)
|
||||
{
|
||||
assert(const_length >= 0);
|
||||
valid_length = const_length;
|
||||
}
|
||||
|
||||
Int64 offset = 0; // start from 1, maybe negative
|
||||
size_t valid_offset = 0; // start from 0, not negative
|
||||
size_t res_offset = 0;
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
size_t input_offset = input_offsets[i - 1];
|
||||
size_t input_bytes = input_offsets[i] - input_offsets[i - 1] - 1;
|
||||
size_t input_size = getSliceSize(&input_data[input_offset], input_bytes);
|
||||
|
||||
size_t replace_offset = replace_offsets[i - 1];
|
||||
size_t replace_bytes = replace_offsets[i] - replace_offsets[i - 1] - 1;
|
||||
size_t replace_size = getSliceSize(&replace_data[replace_offset], replace_bytes);
|
||||
|
||||
if constexpr (offset_is_const)
|
||||
{
|
||||
valid_offset = getValidOffset(const_offset, input_size);
|
||||
}
|
||||
else
|
||||
{
|
||||
offset = column_offset->getInt(i);
|
||||
valid_offset = getValidOffset(offset, input_size);
|
||||
}
|
||||
|
||||
if constexpr (!has_four_args)
|
||||
{
|
||||
valid_length = replace_size;
|
||||
}
|
||||
else if constexpr (!length_is_const)
|
||||
{
|
||||
length = column_length->getInt(i);
|
||||
valid_length = length >= 0 ? length : replace_size;
|
||||
}
|
||||
|
||||
size_t prefix_size = valid_offset;
|
||||
size_t suffix_size = (prefix_size + valid_length > input_size) ? 0 : (input_size - prefix_size - valid_length);
|
||||
|
||||
if constexpr (!is_utf8)
|
||||
{
|
||||
size_t new_res_size = res_data.size() + prefix_size + replace_size + suffix_size + 1; /// +1 for zero terminator
|
||||
res_data.resize(new_res_size);
|
||||
|
||||
/// copy prefix before replaced region
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &input_data[input_offset], prefix_size);
|
||||
res_offset += prefix_size;
|
||||
|
||||
/// copy replace
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &replace_data[replace_offset], replace_size);
|
||||
res_offset += replace_size;
|
||||
|
||||
/// copy suffix after replaced region. It is not necessary to copy if suffix_size is zero.
|
||||
if (suffix_size)
|
||||
{
|
||||
memcpySmallAllowReadWriteOverflow15(
|
||||
&res_data[res_offset], &input_data[input_offset + prefix_size + valid_length], suffix_size);
|
||||
res_offset += suffix_size;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto * input_begin = &input_data[input_offset];
|
||||
const auto * input_end = &input_data[input_offset + input_bytes];
|
||||
const auto * prefix_end = GatherUtils::UTF8StringSource::skipCodePointsForward(input_begin, prefix_size, input_end);
|
||||
size_t prefix_bytes = prefix_end > input_end ? input_bytes : prefix_end - input_begin;
|
||||
const auto * suffix_begin = GatherUtils::UTF8StringSource::skipCodePointsBackward(input_end, suffix_size, input_begin);
|
||||
size_t suffix_bytes = input_end - suffix_begin;
|
||||
size_t new_res_size = res_data.size() + prefix_bytes + replace_bytes + suffix_bytes + 1; /// +1 for zero terminator
|
||||
res_data.resize(new_res_size);
|
||||
|
||||
/// copy prefix before replaced region
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input_begin, prefix_bytes);
|
||||
res_offset += prefix_bytes;
|
||||
|
||||
/// copy replace
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &replace_data[replace_offset], replace_bytes);
|
||||
res_offset += replace_bytes;
|
||||
|
||||
/// copy suffix after replaced region. It is not necessary to copy if suffix_bytes is zero.
|
||||
if (suffix_bytes)
|
||||
{
|
||||
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], suffix_begin, suffix_bytes);
|
||||
res_offset += suffix_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/// add zero terminator
|
||||
res_data[res_offset] = 0;
|
||||
++res_offset;
|
||||
|
||||
res_offsets[i] = res_offset;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(Overlay)
|
||||
{
|
||||
factory.registerFunction<FunctionOverlay<false>>(
|
||||
{.description = R"(
|
||||
Replace a part of a string `input` with another string `replace`, starting at 1-based index `offset`. By default, the number of bytes removed from `input` equals the length of `replace`. If `length` (the optional fourth argument) is specified, a different number of bytes is removed.
|
||||
)",
|
||||
.categories{"String"}},
|
||||
FunctionFactory::Case::Insensitive);
|
||||
|
||||
factory.registerFunction<FunctionOverlay<true>>(
|
||||
{.description = R"(
|
||||
Replace a part of a string `input` with another string `replace`, starting at 1-based index `offset`. By default, the number of characters removed from `input` equals the length of `replace`. If `length` (the optional fourth argument) is specified, a different number of characters is removed.
|
||||
|
||||
Assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
|
||||
)",
|
||||
.categories{"String"}},
|
||||
FunctionFactory::Case::Sensitive);
|
||||
}
|
||||
}
|
||||
@ -1489,10 +1489,8 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
|
||||
size_t size = buf.read(s_pos, remaining_date_size);
|
||||
if (size != remaining_date_size)
|
||||
{
|
||||
s_pos[size] = 0;
|
||||
|
||||
if constexpr (throw_exception)
|
||||
throw Exception(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse DateTime {}", s);
|
||||
throw Exception(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse DateTime {}", std::string_view(s, already_read_length + size));
|
||||
else
|
||||
return false;
|
||||
}
|
||||
@ -1522,10 +1520,8 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
|
||||
|
||||
if (size != time_broken_down_length)
|
||||
{
|
||||
s_pos[size] = 0;
|
||||
|
||||
if constexpr (throw_exception)
|
||||
throw Exception(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse time component of DateTime {}", s);
|
||||
throw Exception(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse time component of DateTime {}", std::string_view(s, size));
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -619,9 +619,18 @@ QueryCache::Writer QueryCache::createWriter(const Key & key, std::chrono::millis
|
||||
return Writer(cache, key, max_entry_size_in_bytes, max_entry_size_in_rows, min_query_runtime, squash_partial_results, max_block_size);
|
||||
}
|
||||
|
||||
void QueryCache::clear()
|
||||
void QueryCache::clear(const std::optional<String> & tag)
|
||||
{
|
||||
if (tag)
|
||||
{
|
||||
auto predicate = [tag](const Key & key, const Cache::MappedPtr &) { return key.tag == tag.value(); };
|
||||
cache.remove(predicate);
|
||||
}
|
||||
else
|
||||
{
|
||||
cache.clear();
|
||||
}
|
||||
|
||||
std::lock_guard lock(mutex);
|
||||
times_executed.clear();
|
||||
}
|
||||
|
||||
@ -211,7 +211,7 @@ public:
|
||||
Reader createReader(const Key & key);
|
||||
Writer createWriter(const Key & key, std::chrono::milliseconds min_query_runtime, bool squash_partial_results, size_t max_block_size, size_t max_query_cache_size_in_bytes_quota, size_t max_query_cache_entries_quota);
|
||||
|
||||
void clear();
|
||||
void clear(const std::optional<String> & tag);
|
||||
|
||||
size_t sizeInBytes() const;
|
||||
size_t count() const;
|
||||
|
||||
@ -3228,12 +3228,12 @@ QueryCachePtr Context::getQueryCache() const
|
||||
return shared->query_cache;
|
||||
}
|
||||
|
||||
void Context::clearQueryCache() const
|
||||
void Context::clearQueryCache(const std::optional<String> & tag) const
|
||||
{
|
||||
std::lock_guard lock(shared->mutex);
|
||||
|
||||
if (shared->query_cache)
|
||||
shared->query_cache->clear();
|
||||
shared->query_cache->clear(tag);
|
||||
}
|
||||
|
||||
void Context::clearCaches() const
|
||||
|
||||
@ -1068,7 +1068,7 @@ public:
|
||||
void setQueryCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_rows);
|
||||
void updateQueryCacheConfiguration(const Poco::Util::AbstractConfiguration & config);
|
||||
std::shared_ptr<QueryCache> getQueryCache() const;
|
||||
void clearQueryCache() const;
|
||||
void clearQueryCache(const std::optional<String> & tag) const;
|
||||
|
||||
/** Clear the caches of the uncompressed blocks and marks.
|
||||
* This is usually done when renaming tables, changing the type of columns, deleting a table.
|
||||
|
||||
@ -9,7 +9,6 @@
|
||||
#include <Parsers/ASTQueryWithOutput.h>
|
||||
#include <Parsers/ASTSelectWithUnionQuery.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -46,7 +45,9 @@ static void applySettingsFromSelectWithUnion(const ASTSelectWithUnionQuery & sel
|
||||
// It is flattened later, when we process UNION ALL/DISTINCT.
|
||||
const auto * last_select = children.back()->as<ASTSelectQuery>();
|
||||
if (last_select && last_select->settings())
|
||||
InterpreterSetQuery(last_select->settings(), context).executeForCurrentContext(/* ignore_setting_constraints= */ false);
|
||||
{
|
||||
InterpreterSetQuery(last_select->settings(), context).executeForCurrentContext();
|
||||
}
|
||||
}
|
||||
|
||||
void InterpreterSetQuery::applySettingsFromQuery(const ASTPtr & ast, ContextMutablePtr context_)
|
||||
@ -54,20 +55,10 @@ void InterpreterSetQuery::applySettingsFromQuery(const ASTPtr & ast, ContextMuta
|
||||
if (!ast)
|
||||
return;
|
||||
|
||||
/// First apply the outermost settings. Then they could be overridden by deeper settings.
|
||||
if (const auto * query_with_output = dynamic_cast<const ASTQueryWithOutput *>(ast.get()))
|
||||
{
|
||||
if (query_with_output->settings_ast)
|
||||
InterpreterSetQuery(query_with_output->settings_ast, context_).executeForCurrentContext(/* ignore_setting_constraints= */ false);
|
||||
|
||||
if (const auto * create_query = ast->as<ASTCreateQuery>(); create_query && create_query->select)
|
||||
applySettingsFromSelectWithUnion(create_query->select->as<ASTSelectWithUnionQuery &>(), context_);
|
||||
}
|
||||
|
||||
if (const auto * select_query = ast->as<ASTSelectQuery>())
|
||||
{
|
||||
if (auto new_settings = select_query->settings())
|
||||
InterpreterSetQuery(new_settings, context_).executeForCurrentContext(/* ignore_setting_constraints= */ false);
|
||||
InterpreterSetQuery(new_settings, context_).executeForCurrentContext();
|
||||
}
|
||||
else if (const auto * select_with_union_query = ast->as<ASTSelectWithUnionQuery>())
|
||||
{
|
||||
@ -76,15 +67,28 @@ void InterpreterSetQuery::applySettingsFromQuery(const ASTPtr & ast, ContextMuta
|
||||
else if (const auto * explain_query = ast->as<ASTExplainQuery>())
|
||||
{
|
||||
if (explain_query->settings_ast)
|
||||
InterpreterSetQuery(explain_query->settings_ast, context_).executeForCurrentContext(/* ignore_setting_constraints= */ false);
|
||||
InterpreterSetQuery(explain_query->settings_ast, context_).executeForCurrentContext();
|
||||
|
||||
applySettingsFromQuery(explain_query->getExplainedQuery(), context_);
|
||||
}
|
||||
else if (const auto * query_with_output = dynamic_cast<const ASTQueryWithOutput *>(ast.get()))
|
||||
{
|
||||
if (query_with_output->settings_ast)
|
||||
InterpreterSetQuery(query_with_output->settings_ast, context_).executeForCurrentContext();
|
||||
|
||||
if (const auto * create_query = ast->as<ASTCreateQuery>())
|
||||
{
|
||||
if (create_query->select)
|
||||
{
|
||||
applySettingsFromSelectWithUnion(create_query->select->as<ASTSelectWithUnionQuery &>(), context_);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (auto * insert_query = ast->as<ASTInsertQuery>())
|
||||
{
|
||||
context_->setInsertFormat(insert_query->format);
|
||||
if (insert_query->settings_ast)
|
||||
InterpreterSetQuery(insert_query->settings_ast, context_).executeForCurrentContext(/* ignore_setting_constraints= */ false);
|
||||
InterpreterSetQuery(insert_query->settings_ast, context_).executeForCurrentContext();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -23,7 +23,7 @@ public:
|
||||
/** Set setting for current context (query context).
|
||||
* It is used for interpretation of SETTINGS clause in SELECT query.
|
||||
*/
|
||||
void executeForCurrentContext(bool ignore_setting_constraints);
|
||||
void executeForCurrentContext(bool ignore_setting_constraints = false);
|
||||
|
||||
bool supportsTransactions() const override { return true; }
|
||||
|
||||
|
||||
@ -369,9 +369,12 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
system_context->clearMMappedFileCache();
|
||||
break;
|
||||
case Type::DROP_QUERY_CACHE:
|
||||
{
|
||||
getContext()->checkAccess(AccessType::SYSTEM_DROP_QUERY_CACHE);
|
||||
getContext()->clearQueryCache();
|
||||
getContext()->clearQueryCache(query.query_cache_tag);
|
||||
break;
|
||||
}
|
||||
|
||||
case Type::DROP_COMPILED_EXPRESSION_CACHE:
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
getContext()->checkAccess(AccessType::SYSTEM_DROP_COMPILED_EXPRESSION_CACHE);
|
||||
|
||||
@ -131,6 +131,8 @@ public:
|
||||
String disk;
|
||||
UInt64 seconds{};
|
||||
|
||||
std::optional<String> query_cache_tag;
|
||||
|
||||
String filesystem_cache_name;
|
||||
std::string key_to_drop;
|
||||
std::optional<size_t> offset_to_drop;
|
||||
|
||||
@ -470,6 +470,7 @@ namespace DB
|
||||
MR_MACROS(TABLE_OVERRIDE, "TABLE OVERRIDE") \
|
||||
MR_MACROS(TABLE, "TABLE") \
|
||||
MR_MACROS(TABLES, "TABLES") \
|
||||
MR_MACROS(TAG, "TAG") \
|
||||
MR_MACROS(TAGS, "TAGS") \
|
||||
MR_MACROS(TAGS_INNER_UUID, "TAGS INNER UUID") \
|
||||
MR_MACROS(TEMPORARY_TABLE, "TEMPORARY TABLE") \
|
||||
|
||||
@ -25,6 +25,7 @@
|
||||
#include <Parsers/ParserTablePropertiesQuery.h>
|
||||
#include <Parsers/ParserWatchQuery.h>
|
||||
#include <Parsers/ParserDescribeCacheQuery.h>
|
||||
#include <Parsers/QueryWithOutputSettingsPushDownVisitor.h>
|
||||
#include <Parsers/Access/ParserShowAccessEntitiesQuery.h>
|
||||
#include <Parsers/Access/ParserShowAccessQuery.h>
|
||||
#include <Parsers/Access/ParserShowCreateAccessEntityQuery.h>
|
||||
@ -151,36 +152,9 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
|
||||
|
||||
}
|
||||
|
||||
/// These two sections are allowed in an arbitrary order.
|
||||
ParserKeyword s_format(Keyword::FORMAT);
|
||||
ParserKeyword s_settings(Keyword::SETTINGS);
|
||||
|
||||
/** Why: let's take the following example:
|
||||
* SELECT 1 UNION ALL SELECT 2 FORMAT TSV
|
||||
* Each subquery can be put in parentheses and have its own settings:
|
||||
* (SELECT 1 SETTINGS a=b) UNION ALL (SELECT 2 SETTINGS c=d) FORMAT TSV
|
||||
* And the whole query can have settings:
|
||||
* (SELECT 1 SETTINGS a=b) UNION ALL (SELECT 2 SETTINGS c=d) FORMAT TSV SETTINGS e=f
|
||||
* A single query with output is parsed in the same way as the UNION ALL chain:
|
||||
* SELECT 1 SETTINGS a=b FORMAT TSV SETTINGS e=f
|
||||
* So while these forms have a slightly different meaning, they both exist:
|
||||
* SELECT 1 SETTINGS a=b FORMAT TSV
|
||||
* SELECT 1 FORMAT TSV SETTINGS e=f
|
||||
* And due to this effect, the users expect that the FORMAT and SETTINGS may go in an arbitrary order.
|
||||
* But while this work:
|
||||
* (SELECT 1) UNION ALL (SELECT 2) FORMAT TSV SETTINGS d=f
|
||||
* This does not work automatically, unless we explicitly allow different orders:
|
||||
* (SELECT 1) UNION ALL (SELECT 2) SETTINGS d=f FORMAT TSV
|
||||
* Inevitably, we also allow this:
|
||||
* SELECT 1 SETTINGS a=b SETTINGS d=f FORMAT TSV
|
||||
* ^^^^^^^^^^^^^^^^^^^^^
|
||||
* Because this part is consumed into ASTSelectWithUnionQuery
|
||||
* and the rest into ASTQueryWithOutput.
|
||||
*/
|
||||
|
||||
for (size_t i = 0; i < 2; ++i)
|
||||
{
|
||||
if (!query_with_output.format && s_format.ignore(pos, expected))
|
||||
if (s_format.ignore(pos, expected))
|
||||
{
|
||||
ParserIdentifier format_p;
|
||||
|
||||
@ -190,16 +164,25 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
|
||||
|
||||
query_with_output.children.push_back(query_with_output.format);
|
||||
}
|
||||
else if (!query_with_output.settings_ast && s_settings.ignore(pos, expected))
|
||||
{
|
||||
|
||||
// SETTINGS key1 = value1, key2 = value2, ...
|
||||
ParserKeyword s_settings(Keyword::SETTINGS);
|
||||
if (!query_with_output.settings_ast && s_settings.ignore(pos, expected))
|
||||
{
|
||||
ParserSetQuery parser_settings(true);
|
||||
if (!parser_settings.parse(pos, query_with_output.settings_ast, expected))
|
||||
return false;
|
||||
query_with_output.children.push_back(query_with_output.settings_ast);
|
||||
|
||||
// SETTINGS after FORMAT is not parsed by the SELECT parser (ParserSelectQuery)
|
||||
// Pass them manually, to apply in InterpreterSelectQuery::initSettings()
|
||||
if (query->as<ASTSelectWithUnionQuery>())
|
||||
{
|
||||
auto settings = query_with_output.settings_ast->clone();
|
||||
assert_cast<ASTSetQuery *>(settings.get())->print_in_format = false;
|
||||
QueryWithOutputSettingsPushDownVisitor::Data data{settings};
|
||||
QueryWithOutputSettingsPushDownVisitor(data).visit(query);
|
||||
}
|
||||
else
|
||||
break;
|
||||
}
|
||||
|
||||
node = std::move(query);
|
||||
|
||||
@ -471,6 +471,16 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &
|
||||
res->seconds = seconds->as<ASTLiteral>()->value.safeGet<UInt64>();
|
||||
break;
|
||||
}
|
||||
case Type::DROP_QUERY_CACHE:
|
||||
{
|
||||
ParserLiteral tag_parser;
|
||||
ASTPtr ast;
|
||||
if (ParserKeyword{Keyword::TAG}.ignore(pos, expected) && tag_parser.parse(pos, ast, expected))
|
||||
res->query_cache_tag = std::make_optional<String>(ast->as<ASTLiteral>()->value.safeGet<String>());
|
||||
if (!parseQueryWithOnCluster(res, pos, expected))
|
||||
return false;
|
||||
break;
|
||||
}
|
||||
case Type::DROP_FILESYSTEM_CACHE:
|
||||
{
|
||||
ParserLiteral path_parser;
|
||||
|
||||
56
src/Parsers/QueryWithOutputSettingsPushDownVisitor.cpp
Normal file
56
src/Parsers/QueryWithOutputSettingsPushDownVisitor.cpp
Normal file
@ -0,0 +1,56 @@
|
||||
#include <Common/SettingsChanges.h>
|
||||
#include <Parsers/QueryWithOutputSettingsPushDownVisitor.h>
|
||||
#include <Parsers/ASTSelectWithUnionQuery.h>
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
#include <Parsers/ASTSubquery.h>
|
||||
|
||||
#include <iterator>
|
||||
#include <algorithm>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
bool QueryWithOutputSettingsPushDownMatcher::needChildVisit(ASTPtr & node, const ASTPtr & child)
|
||||
{
|
||||
if (node->as<ASTSelectWithUnionQuery>())
|
||||
return true;
|
||||
if (node->as<ASTSubquery>())
|
||||
return true;
|
||||
if (child->as<ASTSelectQuery>())
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
void QueryWithOutputSettingsPushDownMatcher::visit(ASTPtr & ast, Data & data)
|
||||
{
|
||||
if (auto * select_query = ast->as<ASTSelectQuery>())
|
||||
visit(*select_query, ast, data);
|
||||
}
|
||||
|
||||
void QueryWithOutputSettingsPushDownMatcher::visit(ASTSelectQuery & select_query, ASTPtr &, Data & data)
|
||||
{
|
||||
ASTPtr select_settings_ast = select_query.settings();
|
||||
if (!select_settings_ast)
|
||||
{
|
||||
select_query.setExpression(ASTSelectQuery::Expression::SETTINGS, data.settings_ast->clone());
|
||||
return;
|
||||
}
|
||||
|
||||
SettingsChanges & select_settings = select_settings_ast->as<ASTSetQuery &>().changes;
|
||||
SettingsChanges & settings = data.settings_ast->as<ASTSetQuery &>().changes;
|
||||
|
||||
for (auto & setting : settings)
|
||||
{
|
||||
auto it = std::find_if(select_settings.begin(), select_settings.end(), [&](auto & select_setting)
|
||||
{
|
||||
return select_setting.name == setting.name;
|
||||
});
|
||||
if (it == select_settings.end())
|
||||
select_settings.push_back(setting);
|
||||
else
|
||||
it->value = setting.value;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
39
src/Parsers/QueryWithOutputSettingsPushDownVisitor.h
Normal file
39
src/Parsers/QueryWithOutputSettingsPushDownVisitor.h
Normal file
@ -0,0 +1,39 @@
|
||||
#pragma once
|
||||
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Interpreters/InDepthNodeVisitor.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ASTSelectQuery;
|
||||
struct SettingChange;
|
||||
class SettingsChanges;
|
||||
|
||||
/// Pushdown SETTINGS clause that goes after FORMAT to the SELECT query:
|
||||
/// (since settings after FORMAT parsed separately not in the ParserSelectQuery but in ParserQueryWithOutput)
|
||||
///
|
||||
/// SELECT 1 FORMAT Null SETTINGS max_block_size = 1 ->
|
||||
/// SELECT 1 SETTINGS max_block_size = 1 FORMAT Null SETTINGS max_block_size = 1
|
||||
///
|
||||
/// Otherwise settings after FORMAT will not be applied.
|
||||
class QueryWithOutputSettingsPushDownMatcher
|
||||
{
|
||||
public:
|
||||
using Visitor = InDepthNodeVisitor<QueryWithOutputSettingsPushDownMatcher, true>;
|
||||
|
||||
struct Data
|
||||
{
|
||||
const ASTPtr & settings_ast;
|
||||
};
|
||||
|
||||
static bool needChildVisit(ASTPtr & node, const ASTPtr & child);
|
||||
static void visit(ASTPtr & ast, Data & data);
|
||||
|
||||
private:
|
||||
static void visit(ASTSelectQuery &, ASTPtr &, Data &);
|
||||
};
|
||||
|
||||
using QueryWithOutputSettingsPushDownVisitor = QueryWithOutputSettingsPushDownMatcher::Visitor;
|
||||
|
||||
}
|
||||
@ -286,6 +286,9 @@ static std::vector<Range> getHyperrectangleForRowGroup(const parquet::FileMetaDa
|
||||
if (!s)
|
||||
continue;
|
||||
|
||||
if (s->descr()->schema_node()->is_repeated())
|
||||
continue;
|
||||
|
||||
auto path = c->path_in_schema()->ToDotVector();
|
||||
if (path.size() != 1)
|
||||
continue; // compound types not supported
|
||||
|
||||
@ -369,7 +369,7 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
|
||||
/// If sample and final are used together no need to calculate sampling expression twice.
|
||||
/// The first time it was calculated for final, because sample key is a part of the PK.
|
||||
/// So, assume that we already have calculated column.
|
||||
ASTPtr sampling_key_ast = metadata_snapshot->getSamplingKeyAST();
|
||||
ASTPtr sampling_key_ast;
|
||||
|
||||
if (final)
|
||||
{
|
||||
@ -377,6 +377,12 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
|
||||
/// We do spoil available_real_columns here, but it is not used later.
|
||||
available_real_columns.emplace_back(sampling_key.column_names[0], std::move(sampling_column_type));
|
||||
}
|
||||
else
|
||||
{
|
||||
sampling_key_ast = metadata_snapshot->getSamplingKeyAST()->clone();
|
||||
}
|
||||
|
||||
chassert(sampling_key_ast != nullptr);
|
||||
|
||||
if (has_lower_limit)
|
||||
{
|
||||
|
||||
@ -67,8 +67,8 @@ struct Settings;
|
||||
M(Bool, fsync_part_directory, false, "Do fsync for part directory after all part operations (writes, renames, etc.).", 0) \
|
||||
M(UInt64, non_replicated_deduplication_window, 0, "How many last blocks of hashes should be kept on disk (0 - disabled).", 0) \
|
||||
M(UInt64, max_parts_to_merge_at_once, 100, "Max amount of parts which can be merged at once (0 - disabled). Doesn't affect OPTIMIZE FINAL query.", 0) \
|
||||
M(UInt64, merge_selecting_sleep_ms, 5000, "Maximum sleep time for merge selecting, a lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters", 0) \
|
||||
M(UInt64, max_merge_selecting_sleep_ms, 60000, "Maximum sleep time for merge selecting, a lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters", 0) \
|
||||
M(UInt64, merge_selecting_sleep_ms, 5000, "Minimum time to wait before trying to select parts to merge again after no parts were selected. A lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters", 0) \
|
||||
M(UInt64, max_merge_selecting_sleep_ms, 60000, "Maximum time to wait before trying to select parts to merge again after no parts were selected. A lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters", 0) \
|
||||
M(Float, merge_selecting_sleep_slowdown_factor, 1.2f, "The sleep time for merge selecting task is multiplied by this factor when there's nothing to merge and divided when a merge was assigned", 0) \
|
||||
M(UInt64, merge_tree_clear_old_temporary_directories_interval_seconds, 60, "The period of executing the clear old temporary directories operation in background.", 0) \
|
||||
M(UInt64, merge_tree_clear_old_parts_interval_seconds, 1, "The period of executing the clear old parts operation in background.", 0) \
|
||||
|
||||
@ -16,3 +16,4 @@ no_implicit_reexport = True
|
||||
strict_equality = True
|
||||
extra_checks = True
|
||||
ignore_missing_imports = True
|
||||
logging-fstring-interpolation = False
|
||||
@ -143,6 +143,8 @@ class DebianArtifactory:
|
||||
print(f" {cmd}")
|
||||
Shell.check(cmd, strict=True)
|
||||
Shell.check("sync")
|
||||
time.sleep(10)
|
||||
Shell.check(f"lsof +D R2MountPoint.MOUNT_POINT", verbose=True)
|
||||
|
||||
def test_packages(self):
|
||||
Shell.check("docker pull ubuntu:latest", strict=True)
|
||||
|
||||
@ -46,6 +46,7 @@ def parse_args():
|
||||
|
||||
MAX_NUMBER_OF_COMMITS_TO_CONSIDER_FOR_RELEASE = 5
|
||||
AUTORELEASE_INFO_FILE = "/tmp/autorelease_info.json"
|
||||
AUTORELEASE_MATRIX_PARAMS = "/tmp/autorelease_params.json"
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
@ -74,6 +75,14 @@ class AutoReleaseInfo:
|
||||
with open(AUTORELEASE_INFO_FILE, "w", encoding="utf-8") as f:
|
||||
print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
|
||||
|
||||
# dump file for GH action matrix that is similar to the file above but with dropped not ready release branches
|
||||
params = dataclasses.asdict(self)
|
||||
params["releases"] = [
|
||||
release for release in params["releases"] if release["ready"]
|
||||
]
|
||||
with open(AUTORELEASE_MATRIX_PARAMS, "w", encoding="utf-8") as f:
|
||||
print(json.dumps(params, indent=2), file=f)
|
||||
|
||||
@staticmethod
|
||||
def from_file() -> "AutoReleaseInfo":
|
||||
with open(AUTORELEASE_INFO_FILE, "r", encoding="utf-8") as json_file:
|
||||
@ -102,7 +111,6 @@ def _prepare(token):
|
||||
refs = list(repo.get_git_matching_refs(f"tags/v{pr.head.ref}"))
|
||||
assert refs
|
||||
|
||||
refs.sort(key=lambda ref: ref.ref)
|
||||
latest_release_tag_ref = refs[-1]
|
||||
latest_release_tag = repo.get_git_tag(latest_release_tag_ref.object.sha)
|
||||
|
||||
@ -110,6 +118,10 @@ def _prepare(token):
|
||||
f"git rev-list --first-parent {latest_release_tag.tag}..origin/{pr.head.ref}",
|
||||
).split("\n")
|
||||
commit_num = len(commits)
|
||||
if latest_release_tag.tag.endswith("new"):
|
||||
print("It's a new release branch - skip auto release for it")
|
||||
continue
|
||||
|
||||
print(
|
||||
f"Previous release [{latest_release_tag.tag}] was [{commit_num}] commits ago, date [{latest_release_tag.tagger.date}]"
|
||||
)
|
||||
@ -133,16 +145,33 @@ def _prepare(token):
|
||||
commits_to_branch_head += 1
|
||||
continue
|
||||
|
||||
commit_ci_status = CI.GH.get_commit_status_by_name(
|
||||
# TODO: switch to check if CI is entirely green
|
||||
statuses = [
|
||||
CI.GH.get_commit_status_by_name(
|
||||
token=token,
|
||||
commit_sha=commit,
|
||||
# handle old name for old releases
|
||||
status_name=(CI.JobNames.BUILD_CHECK, "ClickHouse build check"),
|
||||
)
|
||||
),
|
||||
CI.GH.get_commit_status_by_name(
|
||||
token=token,
|
||||
commit_sha=commit,
|
||||
# handle old name for old releases
|
||||
status_name=CI.JobNames.STATELESS_TEST_RELEASE,
|
||||
),
|
||||
CI.GH.get_commit_status_by_name(
|
||||
token=token,
|
||||
commit_sha=commit,
|
||||
# handle old name for old releases
|
||||
status_name=CI.JobNames.STATEFUL_TEST_RELEASE,
|
||||
),
|
||||
]
|
||||
commit_sha = commit
|
||||
if commit_ci_status == SUCCESS:
|
||||
if any(status == SUCCESS for status in statuses):
|
||||
commit_ci_status = SUCCESS
|
||||
break
|
||||
|
||||
print(f"CI status [{commit_ci_status}] - skip")
|
||||
print(f"CI status [{statuses}] - skip")
|
||||
commits_to_branch_head += 1
|
||||
|
||||
ready = False
|
||||
|
||||
@ -163,7 +163,7 @@ class CI:
|
||||
tidy=True,
|
||||
comment="clang-tidy is used for static analysis",
|
||||
),
|
||||
timeout=10800,
|
||||
timeout=14400,
|
||||
),
|
||||
BuildNames.BINARY_DARWIN: CommonJobConfigs.BUILD.with_properties(
|
||||
build_config=BuildConfig(
|
||||
@ -400,7 +400,10 @@ class CI:
|
||||
required_builds=[BuildNames.PACKAGE_DEBUG], pr_only=True
|
||||
),
|
||||
JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_ASAN], release_only=True, num_batches=4
|
||||
required_builds=[BuildNames.PACKAGE_ASAN],
|
||||
release_only=True,
|
||||
num_batches=4,
|
||||
timeout=10800,
|
||||
),
|
||||
JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_ASAN],
|
||||
@ -535,7 +538,10 @@ class CI:
|
||||
JobNames.FAST_TEST: JobConfig(
|
||||
pr_only=True,
|
||||
digest=DigestConfig(
|
||||
include_paths=["./tests/queries/0_stateless/"],
|
||||
include_paths=[
|
||||
"./tests/queries/0_stateless/",
|
||||
"./tests/docker_scripts/",
|
||||
],
|
||||
exclude_files=[".md"],
|
||||
docker=["clickhouse/fasttest"],
|
||||
),
|
||||
|
||||
@ -415,6 +415,7 @@ class CommonJobConfigs:
|
||||
"./tests/clickhouse-test",
|
||||
"./tests/config",
|
||||
"./tests/*.txt",
|
||||
"./tests/docker_scripts/",
|
||||
],
|
||||
exclude_files=[".md"],
|
||||
docker=["clickhouse/stateless-test"],
|
||||
@ -431,6 +432,7 @@ class CommonJobConfigs:
|
||||
"./tests/clickhouse-test",
|
||||
"./tests/config",
|
||||
"./tests/*.txt",
|
||||
"./tests/docker_scripts/",
|
||||
],
|
||||
exclude_files=[".md"],
|
||||
docker=["clickhouse/stateful-test"],
|
||||
@ -448,6 +450,7 @@ class CommonJobConfigs:
|
||||
"./tests/clickhouse-test",
|
||||
"./tests/config",
|
||||
"./tests/*.txt",
|
||||
"./tests/docker_scripts/",
|
||||
],
|
||||
exclude_files=[".md"],
|
||||
docker=["clickhouse/stress-test"],
|
||||
@ -459,9 +462,9 @@ class CommonJobConfigs:
|
||||
UPGRADE_TEST = JobConfig(
|
||||
job_name_keyword="upgrade",
|
||||
digest=DigestConfig(
|
||||
include_paths=["./tests/ci/upgrade_check.py"],
|
||||
include_paths=["./tests/ci/upgrade_check.py", "./tests/docker_scripts/"],
|
||||
exclude_files=[".md"],
|
||||
docker=["clickhouse/upgrade-check"],
|
||||
docker=["clickhouse/stress-test"],
|
||||
),
|
||||
run_command="upgrade_check.py",
|
||||
runner_type=Runners.STRESS_TESTER,
|
||||
|
||||
@ -102,21 +102,29 @@ class GH:
|
||||
assert len(commit_sha) == 40
|
||||
assert Utils.is_hex(commit_sha)
|
||||
assert not Utils.is_hex(token)
|
||||
url = f"https://api.github.com/repos/{Envs.GITHUB_REPOSITORY}/commits/{commit_sha}/statuses?per_page={200}"
|
||||
|
||||
url = f"https://api.github.com/repos/{Envs.GITHUB_REPOSITORY}/commits/{commit_sha}/statuses"
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.get(url, headers=headers, timeout=5)
|
||||
|
||||
if isinstance(status_name, str):
|
||||
status_name = (status_name,)
|
||||
|
||||
while url:
|
||||
response = requests.get(url, headers=headers, timeout=5)
|
||||
if response.status_code == 200:
|
||||
assert "next" not in response.links, "Response truncated"
|
||||
statuses = response.json()
|
||||
for status in statuses:
|
||||
if status["context"] in status_name:
|
||||
return status["state"] # type: ignore
|
||||
|
||||
# Check if there is a next page
|
||||
url = response.links.get("next", {}).get("url")
|
||||
else:
|
||||
break
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
|
||||
@ -93,7 +93,7 @@ def process_single_image(
|
||||
results = [] # type: TestResults
|
||||
for ver in versions:
|
||||
stopwatch = Stopwatch()
|
||||
for i in range(5):
|
||||
for i in range(2):
|
||||
success, build_log = build_and_push_one_image(
|
||||
image, ver, additional_cache, push, from_tag
|
||||
)
|
||||
|
||||
@ -31,15 +31,14 @@ def get_fasttest_cmd(
|
||||
"--security-opt seccomp=unconfined " # required to issue io_uring sys-calls
|
||||
"--network=host " # required to get access to IAM credentials
|
||||
f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output "
|
||||
f"-e FASTTEST_SOURCE=/ClickHouse "
|
||||
f"-e FASTTEST_SOURCE=/repo "
|
||||
f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' "
|
||||
f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} "
|
||||
f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 "
|
||||
f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache "
|
||||
"-e stage=clone_submodules "
|
||||
f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse "
|
||||
f"--volume={repo_path}/tests/analyzer_tech_debt.txt:/analyzer_tech_debt.txt "
|
||||
f"--volume={output_path}:/test_output {image}"
|
||||
f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/repo "
|
||||
f"--volume={output_path}:/test_output {image} /repo/tests/docker_scripts/fasttest_runner.sh"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@ -119,24 +119,24 @@ def get_run_command(
|
||||
envs += [f"-e {e}" for e in additional_envs]
|
||||
|
||||
env_str = " ".join(envs)
|
||||
volume_with_broken_test = (
|
||||
f"--volume={repo_path}/tests/analyzer_tech_debt.txt:/analyzer_tech_debt.txt "
|
||||
if "analyzer" not in check_name
|
||||
else ""
|
||||
)
|
||||
|
||||
if "stateful" in check_name.lower():
|
||||
run_script = "/repo/tests/docker_scripts/stateful_runner.sh"
|
||||
elif "stateless" in check_name.lower():
|
||||
run_script = "/repo/tests/docker_scripts/stateless_runner.sh"
|
||||
else:
|
||||
assert False
|
||||
|
||||
return (
|
||||
f"docker run --rm --name func-tester --volume={builds_path}:/package_folder "
|
||||
# For dmesg and sysctl
|
||||
"--privileged "
|
||||
f"{ci_logs_args}"
|
||||
f"--volume={repo_path}/tests:/usr/share/clickhouse-test "
|
||||
f"--volume={repo_path}/utils/grpc-client:/usr/share/clickhouse-utils/grpc-client "
|
||||
f"{volume_with_broken_test}"
|
||||
f"{ci_logs_args} "
|
||||
f"--volume={repo_path}:/repo "
|
||||
f"--volume={result_path}:/test_output "
|
||||
f"--volume={server_log_path}:/var/log/clickhouse-server "
|
||||
"--security-opt seccomp=unconfined " # required to issue io_uring sys-calls
|
||||
f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}"
|
||||
f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image} {run_script}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@ -57,10 +57,16 @@ def get_run_command(
|
||||
additional_envs: List[str],
|
||||
ci_logs_args: str,
|
||||
image: DockerImage,
|
||||
upgrade_check: bool,
|
||||
) -> str:
|
||||
envs = [f"-e {e}" for e in additional_envs]
|
||||
env_str = " ".join(envs)
|
||||
|
||||
if upgrade_check:
|
||||
run_script = "/repo/tests/docker_scripts/upgrade_runner.sh"
|
||||
else:
|
||||
run_script = "/repo/tests/docker_scripts/stress_runner.sh"
|
||||
|
||||
cmd = (
|
||||
"docker run --cap-add=SYS_PTRACE "
|
||||
# For dmesg and sysctl
|
||||
@ -70,8 +76,8 @@ def get_run_command(
|
||||
f"{ci_logs_args}"
|
||||
f"--volume={build_path}:/package_folder "
|
||||
f"--volume={result_path}:/test_output "
|
||||
f"--volume={repo_tests_path}:/usr/share/clickhouse-test "
|
||||
f"--volume={server_log_path}:/var/log/clickhouse-server {env_str} {image} "
|
||||
f"--volume={repo_tests_path}/..:/repo "
|
||||
f"--volume={server_log_path}:/var/log/clickhouse-server {env_str} {image} {run_script}"
|
||||
)
|
||||
|
||||
return cmd
|
||||
@ -128,7 +134,7 @@ def process_results(
|
||||
return state, description, test_results, additional_files
|
||||
|
||||
|
||||
def run_stress_test(docker_image_name: str) -> None:
|
||||
def run_stress_test(upgrade_check: bool = False) -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
for handler in logging.root.handlers:
|
||||
# pylint: disable=protected-access
|
||||
@ -148,7 +154,7 @@ def run_stress_test(docker_image_name: str) -> None:
|
||||
|
||||
pr_info = PRInfo()
|
||||
|
||||
docker_image = pull_image(get_docker_image(docker_image_name))
|
||||
docker_image = pull_image(get_docker_image("clickhouse/stress-test"))
|
||||
|
||||
packages_path = temp_path / "packages"
|
||||
packages_path.mkdir(parents=True, exist_ok=True)
|
||||
@ -177,6 +183,7 @@ def run_stress_test(docker_image_name: str) -> None:
|
||||
additional_envs,
|
||||
ci_logs_args,
|
||||
docker_image,
|
||||
upgrade_check,
|
||||
)
|
||||
logging.info("Going to run stress test: %s", run_command)
|
||||
|
||||
@ -208,4 +215,4 @@ def run_stress_test(docker_image_name: str) -> None:
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_stress_test("clickhouse/stress-test")
|
||||
run_stress_test()
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import stress_check
|
||||
|
||||
if __name__ == "__main__":
|
||||
stress_check.run_stress_test("clickhouse/upgrade-check")
|
||||
stress_check.run_stress_test(upgrade_check=True)
|
||||
|
||||
@ -39,6 +39,7 @@ from errno import ESRCH
|
||||
from subprocess import PIPE, Popen
|
||||
from time import sleep, time
|
||||
from typing import Dict, List, Optional, Set, Tuple, Union
|
||||
from ast import literal_eval as make_tuple
|
||||
|
||||
try:
|
||||
import termcolor # type: ignore
|
||||
@ -1145,9 +1146,24 @@ class TestCase:
|
||||
|
||||
return description + "\n"
|
||||
|
||||
def apply_random_settings_limits(self, random_settings):
|
||||
for setting in random_settings:
|
||||
if setting in self.random_settings_limits:
|
||||
min_value = self.random_settings_limits[setting][0]
|
||||
if min_value and random_settings[setting] < min_value:
|
||||
random_settings[setting] = min_value
|
||||
max_value = self.random_settings_limits[setting][1]
|
||||
if max_value and random_settings[setting] > max_value:
|
||||
random_settings[setting] = max_value
|
||||
|
||||
def __init__(self, suite, case: str, args, is_concurrent: bool):
|
||||
self.case: str = case # case file name
|
||||
self.tags: Set[str] = suite.all_tags[case] if case in suite.all_tags else set()
|
||||
self.random_settings_limits = (
|
||||
suite.all_random_settings_limits[case]
|
||||
if case in suite.all_random_settings_limits
|
||||
else {}
|
||||
)
|
||||
|
||||
for tag in os.getenv("GLOBAL_TAGS", "").split(","):
|
||||
self.tags.add(tag.strip())
|
||||
@ -1189,11 +1205,13 @@ class TestCase:
|
||||
|
||||
if self.randomize_settings:
|
||||
self.random_settings = SettingsRandomizer.get_random_settings(args)
|
||||
self.apply_random_settings_limits(self.random_settings)
|
||||
|
||||
if self.randomize_merge_tree_settings:
|
||||
self.merge_tree_random_settings = (
|
||||
MergeTreeSettingsRandomizer.get_random_settings(args)
|
||||
)
|
||||
self.apply_random_settings_limits(self.merge_tree_random_settings)
|
||||
|
||||
self.base_url_params = (
|
||||
os.environ["CLICKHOUSE_URL_PARAMS"]
|
||||
@ -1963,7 +1981,9 @@ class TestSuite:
|
||||
return test_name
|
||||
|
||||
@staticmethod
|
||||
def read_test_tags(suite_dir: str, all_tests: List[str]) -> Dict[str, Set[str]]:
|
||||
def read_test_tags_and_random_settings_limits(
|
||||
suite_dir: str, all_tests: List[str]
|
||||
) -> (Dict[str, Set[str]], Dict[str, Dict[str, Tuple[int, int]]]):
|
||||
def get_comment_sign(filename):
|
||||
if filename.endswith(".sql") or filename.endswith(".sql.j2"):
|
||||
return "--"
|
||||
@ -1988,27 +2008,58 @@ class TestSuite:
|
||||
tags = {tag.strip() for tag in tags}
|
||||
return tags
|
||||
|
||||
def is_shebang(line: str) -> bool:
|
||||
return line.startswith("#!")
|
||||
def parse_random_settings_limits_from_line(
|
||||
line, comment_sign
|
||||
) -> Dict[str, Tuple[int, int]]:
|
||||
if not line.startswith(comment_sign):
|
||||
return {}
|
||||
random_settings_limits_str = line[len(comment_sign) :].lstrip()
|
||||
random_settings_limits_prefix = "Random settings limits:"
|
||||
if not random_settings_limits_str.startswith(random_settings_limits_prefix):
|
||||
return {}
|
||||
random_settings_limits_str = random_settings_limits_str[
|
||||
len(random_settings_limits_prefix) :
|
||||
]
|
||||
# limits are specified in a form 'setting1=(min, max); setting2=(min,max); ...'
|
||||
random_settings_limits = {}
|
||||
for setting_and_limit in random_settings_limits_str.split(";"):
|
||||
setting_and_limit = setting_and_limit.split("=")
|
||||
random_settings_limits[setting_and_limit[0].strip()] = make_tuple(
|
||||
setting_and_limit[1]
|
||||
)
|
||||
return random_settings_limits
|
||||
|
||||
def find_tag_line(file):
|
||||
for line in file:
|
||||
line = line.strip()
|
||||
if line and not is_shebang(line):
|
||||
def find_tag_line(lines, comment_sign):
|
||||
for line in lines:
|
||||
if line.startswith(comment_sign) and line[
|
||||
len(comment_sign) :
|
||||
].lstrip().startswith("Tags:"):
|
||||
return line
|
||||
return ""
|
||||
|
||||
def load_tags_from_file(filepath):
|
||||
def find_random_settings_limits_line(lines, comment_sign):
|
||||
for line in lines:
|
||||
if line.startswith(comment_sign) and line[
|
||||
len(comment_sign) :
|
||||
].lstrip().startswith("Random settings limits:"):
|
||||
return line
|
||||
return ""
|
||||
|
||||
def load_tags_and_random_settings_limits_from_file(filepath):
|
||||
comment_sign = get_comment_sign(filepath)
|
||||
need_query_params = False
|
||||
with open(filepath, "r", encoding="utf-8") as file:
|
||||
try:
|
||||
tag_line = find_tag_line(file)
|
||||
lines = file.readlines()
|
||||
tag_line = find_tag_line(lines, comment_sign)
|
||||
random_settings_limits_line = find_random_settings_limits_line(
|
||||
lines, comment_sign
|
||||
)
|
||||
except UnicodeDecodeError:
|
||||
return []
|
||||
return [], {}
|
||||
try:
|
||||
if filepath.endswith(".sql"):
|
||||
for line in file:
|
||||
for line in lines:
|
||||
if "{CLICKHOUSE_DATABASE" in line:
|
||||
need_query_params = True
|
||||
except UnicodeDecodeError:
|
||||
@ -2016,18 +2067,31 @@ class TestSuite:
|
||||
parsed_tags = parse_tags_from_line(tag_line, comment_sign)
|
||||
if need_query_params:
|
||||
parsed_tags.add("need-query-parameters")
|
||||
return parsed_tags
|
||||
random_settings_limits = parse_random_settings_limits_from_line(
|
||||
random_settings_limits_line, comment_sign
|
||||
)
|
||||
return parsed_tags, random_settings_limits
|
||||
|
||||
all_tags = {}
|
||||
all_random_settings_limits = {}
|
||||
start_time = datetime.now()
|
||||
for test_name in all_tests:
|
||||
tags = load_tags_from_file(os.path.join(suite_dir, test_name))
|
||||
(
|
||||
tags,
|
||||
random_settings_limits,
|
||||
) = load_tags_and_random_settings_limits_from_file(
|
||||
os.path.join(suite_dir, test_name)
|
||||
) # noqa: ignore E203
|
||||
if tags:
|
||||
all_tags[test_name] = tags
|
||||
if random_settings_limits:
|
||||
all_random_settings_limits[test_name] = random_settings_limits
|
||||
elapsed = (datetime.now() - start_time).total_seconds()
|
||||
if elapsed > 1:
|
||||
print(f"Tags for suite {suite_dir} read in {elapsed:.2f} seconds")
|
||||
return all_tags
|
||||
print(
|
||||
f"Tags and random settings limits for suite {suite_dir} read in {elapsed:.2f} seconds"
|
||||
)
|
||||
return all_tags, all_random_settings_limits
|
||||
|
||||
def __init__(self, args, suite_path: str, suite_tmp_path: str, suite: str):
|
||||
self.args = args
|
||||
@ -2057,10 +2121,16 @@ class TestSuite:
|
||||
self.all_tests: List[str] = self.get_tests_list(
|
||||
self.tests_in_suite_key_func, filter_func
|
||||
)
|
||||
self.all_tags: Dict[str, Set[str]] = self.read_test_tags(
|
||||
|
||||
all_tags_and_random_settings_limits = (
|
||||
self.read_test_tags_and_random_settings_limits(
|
||||
self.suite_path, self.all_tests
|
||||
)
|
||||
|
||||
)
|
||||
self.all_tags: Dict[str, Set[str]] = all_tags_and_random_settings_limits[0]
|
||||
self.all_random_settings_limits: Dict[str, Dict[str, (int, int)]] = (
|
||||
all_tags_and_random_settings_limits[1]
|
||||
)
|
||||
self.sequential_tests = []
|
||||
self.parallel_tests = []
|
||||
for test_name in self.all_tests:
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
# shellcheck source=./utils.lib
|
||||
source /utils.lib
|
||||
source /repo/tests/docker_scripts/utils.lib
|
||||
|
||||
function attach_gdb_to_clickhouse()
|
||||
{
|
||||
@ -325,7 +325,7 @@ case "$stage" in
|
||||
;&
|
||||
"run_tests")
|
||||
run_tests ||:
|
||||
/process_functional_tests_result.py --in-results-dir "$FASTTEST_OUTPUT/" \
|
||||
/repo/tests/docker_scripts/process_functional_tests_result.py --in-results-dir "$FASTTEST_OUTPUT/" \
|
||||
--out-results-file "$FASTTEST_OUTPUT/test_results.tsv" \
|
||||
--out-status-file "$FASTTEST_OUTPUT/check_status.tsv" || echo -e "failure\tCannot parse results" > "$FASTTEST_OUTPUT/check_status.tsv"
|
||||
;;
|
||||
@ -32,7 +32,7 @@ def process_test_log(log_path, broken_tests):
|
||||
success_finish = False
|
||||
test_results = []
|
||||
test_end = True
|
||||
with open(log_path, "r") as test_file:
|
||||
with open(log_path, "r", encoding="utf-8") as test_file:
|
||||
for line in test_file:
|
||||
original_line = line
|
||||
line = line.strip()
|
||||
@ -150,7 +150,7 @@ def process_result(result_path, broken_tests):
|
||||
|
||||
if result_path and os.path.exists(result_path):
|
||||
(
|
||||
total,
|
||||
_total,
|
||||
skipped,
|
||||
unknown,
|
||||
failed,
|
||||
@ -191,11 +191,11 @@ def process_result(result_path, broken_tests):
|
||||
else:
|
||||
description = ""
|
||||
|
||||
description += "fail: {}, passed: {}".format(failed, success)
|
||||
description += f"fail: {failed}, passed: {success}"
|
||||
if skipped != 0:
|
||||
description += ", skipped: {}".format(skipped)
|
||||
description += f", skipped: {skipped}"
|
||||
if unknown != 0:
|
||||
description += ", unknown: {}".format(unknown)
|
||||
description += f", unknown: {unknown}"
|
||||
else:
|
||||
state = "failure"
|
||||
description = "Output log doesn't exist"
|
||||
@ -205,10 +205,10 @@ def process_result(result_path, broken_tests):
|
||||
|
||||
|
||||
def write_results(results_file, status_file, results, status):
|
||||
with open(results_file, "w") as f:
|
||||
with open(results_file, "w", encoding="utf-8") as f:
|
||||
out = csv.writer(f, delimiter="\t")
|
||||
out.writerows(results)
|
||||
with open(status_file, "w") as f:
|
||||
with open(status_file, "w", encoding="utf-8") as f:
|
||||
out = csv.writer(f, delimiter="\t")
|
||||
out.writerow(status)
|
||||
|
||||
@ -221,15 +221,15 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--in-results-dir", default="/test_output/")
|
||||
parser.add_argument("--out-results-file", default="/test_output/test_results.tsv")
|
||||
parser.add_argument("--out-status-file", default="/test_output/check_status.tsv")
|
||||
parser.add_argument("--broken-tests", default="/analyzer_tech_debt.txt")
|
||||
parser.add_argument("--broken-tests", default="/repo/tests/analyzer_tech_debt.txt")
|
||||
args = parser.parse_args()
|
||||
|
||||
broken_tests = list()
|
||||
broken_tests = []
|
||||
if os.path.exists(args.broken_tests):
|
||||
logging.info(f"File {args.broken_tests} with broken tests found")
|
||||
with open(args.broken_tests) as f:
|
||||
print(f"File {args.broken_tests} with broken tests found")
|
||||
with open(args.broken_tests, encoding="utf-8") as f:
|
||||
broken_tests = f.read().splitlines()
|
||||
logging.info(f"Broken tests in the list: {len(broken_tests)}")
|
||||
print(f"Broken tests in the list: {len(broken_tests)}")
|
||||
|
||||
state, description, test_results = process_result(args.in_results_dir, broken_tests)
|
||||
logging.info("Result parsed")
|
||||
@ -5,7 +5,7 @@ set -e -x -a -u
|
||||
|
||||
ls -lha
|
||||
|
||||
cd hadoop-3.3.1
|
||||
cd /hadoop-3.3.1
|
||||
|
||||
export JAVA_HOME=/usr
|
||||
mkdir -p target/test/data
|
||||
@ -143,7 +143,7 @@ main() {
|
||||
fi
|
||||
start_minio
|
||||
setup_minio "$1"
|
||||
upload_data "${query_dir}" "${2:-/usr/share/clickhouse-test}"
|
||||
upload_data "${query_dir}" "${2:-/repo/tests/}"
|
||||
setup_aws_credentials
|
||||
}
|
||||
|
||||
@ -14,17 +14,17 @@ dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
|
||||
dpkg -i package_folder/clickhouse-server_*.deb
|
||||
dpkg -i package_folder/clickhouse-client_*.deb
|
||||
|
||||
ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
|
||||
ln -s /repo/tests/clickhouse-test /usr/bin/clickhouse-test
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /utils.lib
|
||||
source /repo/tests/docker_scripts/utils.lib
|
||||
|
||||
# install test configs
|
||||
/usr/share/clickhouse-test/config/install.sh
|
||||
/repo/tests/config/install.sh
|
||||
|
||||
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence &
|
||||
|
||||
./setup_minio.sh stateful
|
||||
/repo/tests/docker_scripts/setup_minio.sh stateful
|
||||
./mc admin trace clickminio > /test_output/minio.log &
|
||||
MC_ADMIN_PID=$!
|
||||
|
||||
@ -105,7 +105,7 @@ setup_logs_replication
|
||||
|
||||
clickhouse-client --query "SHOW DATABASES"
|
||||
clickhouse-client --query "CREATE DATABASE datasets"
|
||||
clickhouse-client --multiquery < create.sql
|
||||
clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql
|
||||
clickhouse-client --query "SHOW TABLES FROM datasets"
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
@ -237,6 +237,7 @@ function run_tests()
|
||||
--hung-check
|
||||
--print-time
|
||||
--capture-client-stacktrace
|
||||
--queries "/repo/tests/queries"
|
||||
"${ADDITIONAL_OPTIONS[@]}"
|
||||
"$SKIP_TESTS_OPTION"
|
||||
)
|
||||
@ -259,7 +260,7 @@ ls -la ./
|
||||
echo "Files in root directory"
|
||||
ls -la /
|
||||
|
||||
/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
/repo/tests/docker_scripts/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
|
||||
sudo clickhouse stop ||:
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
@ -1,10 +1,13 @@
|
||||
#!/bin/bash
|
||||
|
||||
# fail on errors, verbose and export all env variables
|
||||
set -e -x -a
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /setup_export_logs.sh
|
||||
|
||||
# shellcheck source=../stateless/stress_tests.lib
|
||||
source /stress_tests.lib
|
||||
source /repo/tests/docker_scripts/stress_tests.lib
|
||||
|
||||
# Avoid overlaps with previous runs
|
||||
dmesg --clear
|
||||
@ -39,20 +42,22 @@ if [[ -z "$BUGFIX_VALIDATE_CHECK" ]]; then
|
||||
chc --version || exit 1
|
||||
fi
|
||||
|
||||
ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
|
||||
ln -sf /repo/tests/clickhouse-test /usr/bin/clickhouse-test
|
||||
|
||||
export CLICKHOUSE_GRPC_CLIENT="/repo/utils/grpc-client/clickhouse-grpc-client.py"
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /attach_gdb.lib
|
||||
source /repo/tests/docker_scripts/attach_gdb.lib
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /utils.lib
|
||||
source /repo/tests/docker_scripts/utils.lib
|
||||
|
||||
# install test configs
|
||||
/usr/share/clickhouse-test/config/install.sh
|
||||
/repo/tests/config/install.sh
|
||||
|
||||
./setup_minio.sh stateless
|
||||
/repo/tests/docker_scripts/setup_minio.sh stateless
|
||||
|
||||
./setup_hdfs_minicluster.sh
|
||||
/repo/tests/docker_scripts/setup_hdfs_minicluster.sh
|
||||
|
||||
config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml
|
||||
|
||||
@ -75,6 +80,9 @@ fi
|
||||
|
||||
export IS_FLAKY_CHECK=0
|
||||
|
||||
# Export NUM_TRIES so python scripts will see its value as env variable
|
||||
export NUM_TRIES
|
||||
|
||||
# For flaky check we also enable thread fuzzer
|
||||
if [ "$NUM_TRIES" -gt "1" ]; then
|
||||
export IS_FLAKY_CHECK=1
|
||||
@ -316,6 +324,7 @@ function run_tests()
|
||||
--print-time
|
||||
--no-drop-if-fail
|
||||
--capture-client-stacktrace
|
||||
--queries "/repo/tests/queries"
|
||||
--test-runs "$NUM_TRIES"
|
||||
"${ADDITIONAL_OPTIONS[@]}"
|
||||
)
|
||||
@ -341,7 +350,7 @@ ls -la ./
|
||||
echo "Files in root directory"
|
||||
ls -la /
|
||||
|
||||
/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
/repo/tests/docker_scripts/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
|
||||
clickhouse-client -q "system flush logs" ||:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user