thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:ClickHouse/ClickHouse into BLAKE3

Browse Source
This commit is contained in:
BoloniniD 2022-04-22 18:12:54 +03:00
commit a26acd3660
96 changed files with 1499 additions and 546 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
.github/workflows/backport_branches.yml vendored
View File

@ -142,11 +142,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -181,23 +180,26 @@ jobs:
uses: actions/download-artifact@v2
with:
name: changed_images
path: ${{ runner.temp }}/images_path
path: ${{ env.IMAGES_PATH }}
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -228,11 +230,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -273,11 +274,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -318,11 +318,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

69
.github/workflows/master.yml vendored
View File

@ -207,10 +207,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -249,10 +250,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -293,10 +295,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
fetch-depth: 0 # is needed for ancestor commit search
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -338,10 +341,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -382,11 +386,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -427,11 +430,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -472,11 +474,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -517,11 +518,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -562,11 +562,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -607,11 +606,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -655,11 +653,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -700,11 +697,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -746,10 +742,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -791,10 +788,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -836,10 +834,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -881,10 +880,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -926,10 +926,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

1
.github/workflows/nightly.yml vendored
View File

@ -100,7 +100,6 @@ jobs:
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"

95
.github/workflows/pull_request.yml vendored
View File

@ -265,11 +265,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -311,10 +310,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
fetch-depth: 0 # is needed for ancestor commit search
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -355,11 +355,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -395,13 +394,15 @@ jobs:
with:
name: changed_images
path: ${{ runner.temp }}/images_path
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -411,7 +412,7 @@ jobs:
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -437,13 +438,15 @@ jobs:
with:
name: changed_images
path: ${{ runner.temp }}/images_path
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -453,7 +456,7 @@ jobs:
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -484,11 +487,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -529,11 +531,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -574,11 +575,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -619,11 +619,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -664,11 +663,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -712,11 +710,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -757,11 +754,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -802,11 +798,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -847,11 +842,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -892,11 +886,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -937,11 +930,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -982,11 +974,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

31
.github/workflows/release_branches.yml vendored
View File

@ -134,10 +134,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -176,10 +177,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -219,11 +221,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -264,11 +265,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -309,11 +309,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -354,11 +353,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -399,11 +397,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

153
CHANGELOG.md
View File

@ -1,9 +1,162 @@
### Table of Contents
**[ClickHouse release v22.4, 2022-04-20](#224)**<br>
**[ClickHouse release v22.3-lts, 2022-03-17](#223)**<br>
**[ClickHouse release v22.2, 2022-02-17](#222)**<br>
**[ClickHouse release v22.1, 2022-01-18](#221)**<br>
**[Changelog for 2021](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/whats-new/changelog/2021.md)**<br>
### <a id="224"></a> ClickHouse release master FIXME as compared to v22.3.3.44-lts
#### Backward Incompatible Change
* Do not allow SETTINGS after FORMAT for INSERT queries (there is compatibility setting `parser_settings_after_format_compact` to accept such queries, but it is turned OFF by default). [#35883](https://github.com/ClickHouse/ClickHouse/pull/35883) ([Azat Khuzhin](https://github.com/azat)).
* Function `yandexConsistentHash` (consistent hashing algorithm by Konstantin "kostik" Oblakov) is renamed to `kostikConsistentHash`. The old name is left as an alias for compatibility. Although this change is backward compatible, we may remove the alias in subsequent releases, that's why it's recommended to update the usages of this function in your apps. [#35553](https://github.com/ClickHouse/ClickHouse/pull/35553) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
#### New Feature
* Added INTERPOLATE extension to the ORDER BY ... WITH FILL. Closes [#34903](https://github.com/ClickHouse/ClickHouse/issues/34903). [#35349](https://github.com/ClickHouse/ClickHouse/pull/35349) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Profiling on Processors level (under `log_processors_profiles` setting, ClickHouse will write time that processor spent during execution/waiting for data to `system.processors_profile_log` table). [#34355](https://github.com/ClickHouse/ClickHouse/pull/34355) ([Azat Khuzhin](https://github.com/azat)).
* Added functions makeDate(year, month, day), makeDate32(year, month, day). [#35628](https://github.com/ClickHouse/ClickHouse/pull/35628) ([Alexander Gololobov](https://github.com/davenger)). Implementation of makeDateTime() and makeDateTIme64(). [#35934](https://github.com/ClickHouse/ClickHouse/pull/35934) ([Alexander Gololobov](https://github.com/davenger)).
* Support new type of quota `WRITTEN BYTES` to limit amount of written bytes during insert queries. [#35736](https://github.com/ClickHouse/ClickHouse/pull/35736) ([Anton Popov](https://github.com/CurtizJ)).
* Added function `flattenTuple`. It receives nested named `Tuple` as an argument and returns a flatten `Tuple` which elements are the paths from the original `Tuple`. E.g.: `Tuple(a Int, Tuple(b Int, c Int)) -> Tuple(a Int, b Int, c Int)`. `flattenTuple` can be used to select all paths from type `Object` as separate columns. [#35690](https://github.com/ClickHouse/ClickHouse/pull/35690) ([Anton Popov](https://github.com/CurtizJ)).
* Added functions `arrayFirstOrNull`, `arrayLastOrNull`. Closes [#35238](https://github.com/ClickHouse/ClickHouse/issues/35238). [#35414](https://github.com/ClickHouse/ClickHouse/pull/35414) ([Maksim Kita](https://github.com/kitaisreal)).
* Added functions `minSampleSizeContinous` and `minSampleSizeConversion`. Author [achimbab](https://github.com/achimbab). [#35360](https://github.com/ClickHouse/ClickHouse/pull/35360) ([Maksim Kita](https://github.com/kitaisreal)).
* New functions minSampleSizeContinous and minSampleSizeConversion. [#34354](https://github.com/ClickHouse/ClickHouse/pull/34354) ([achimbab](https://github.com/achimbab)).
* Introduce format `ProtobufList` (all records as repeated messages in out Protobuf). Closes [#16436](https://github.com/ClickHouse/ClickHouse/issues/16436). [#35152](https://github.com/ClickHouse/ClickHouse/pull/35152) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Add `h3PointDistM`, `h3PointDistKm`, `h3PointDistRads`, `h3GetRes0Indexes`, `h3GetPentagonIndexes` functions. [#34568](https://github.com/ClickHouse/ClickHouse/pull/34568) ([Bharat Nallan](https://github.com/bharatnc)).
* Add `toLastDayOfMonth` function which rounds up a date or date with time to the last day of the month. [#33501](https://github.com/ClickHouse/ClickHouse/issues/33501). [#34394](https://github.com/ClickHouse/ClickHouse/pull/34394) ([Habibullah Oladepo](https://github.com/holadepo)).
* New aggregation function groupSortedArray to obtain an array of first N values. [#34055](https://github.com/ClickHouse/ClickHouse/pull/34055) ([palegre-tiny](https://github.com/palegre-tiny)).
* Added load balancing setting for \[Zoo\]Keeper client. Closes [#29617](https://github.com/ClickHouse/ClickHouse/issues/29617). [#30325](https://github.com/ClickHouse/ClickHouse/pull/30325) ([小路](https://github.com/nicelulu)).
* Add a new kind of row policies named `simple`. Before this PR we had two kinds or row policies: `permissive` and `restrictive`. A `simple` row policy adds a new filter on a table without any side-effects like it was for permissive and restrictive policies. [#35345](https://github.com/ClickHouse/ClickHouse/pull/35345) ([Vitaly Baranov](https://github.com/vitlibar)).
* Added an ability to specify cluster secret in replicated database. [#35333](https://github.com/ClickHouse/ClickHouse/pull/35333) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
* Added sanity checks on server startup (available memory and disk space, max thread count, etc). [#34566](https://github.com/ClickHouse/ClickHouse/pull/34566) ([Sergei Trifonov](https://github.com/serxa)).
* INTERVAL improvement - can be used with `[MILLI|MICRO|NANO]SECOND`. Added `toStartOf[Milli|Micro|Nano]second()` functions. Added `[add|subtract][Milli|Micro|Nano]seconds()`. [#34353](https://github.com/ClickHouse/ClickHouse/pull/34353) ([Andrey Zvonov](https://github.com/zvonand)).
#### Experimental Feature
* Added support for transactions for simple `MergeTree` tables. This feature is highly experimental and not recommended for production. Part of [#22086](https://github.com/ClickHouse/ClickHouse/issues/22086). [#24258](https://github.com/ClickHouse/ClickHouse/pull/24258) ([tavplubix](https://github.com/tavplubix)).
* Support schema inference for type `Object` in format `JSONEachRow`. Allow to convert columns of type `Map` to columns of type `Object`. [#35629](https://github.com/ClickHouse/ClickHouse/pull/35629) ([Anton Popov](https://github.com/CurtizJ)).
* Allow to write remote FS cache on all write operations. Add `system.remote_filesystem_cache` table. Add `drop remote filesystem cache` query. Add introspection for s3 metadata with `system.remote_data_paths` table. Closes [#34021](https://github.com/ClickHouse/ClickHouse/issues/34021). Add cache option for merges by adding mode `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` (turned on by default for merges and can also be turned on by query setting with the same name). Rename cache related settings (`remote_fs_enable_cache -> enable_filesystem_cache`, etc). [#35475](https://github.com/ClickHouse/ClickHouse/pull/35475) ([Kseniia Sumarokova](https://github.com/kssenii)).
* An option to store parts metadata in RocksDB. Speed up parts loading process of MergeTree to accelerate starting up of clickhouse-server. With this improvement, clickhouse-server was able to decrease starting up time from 75 minutes to 20 seconds, with 700k mergetree parts. [#32928](https://github.com/ClickHouse/ClickHouse/pull/32928) ([李扬](https://github.com/taiyang-li)).
#### Performance Improvement
* A new query plan optimization. Evaluate functions after `ORDER BY` when possible. As an example, for a query `SELECT sipHash64(number) FROM numbers(1e8) ORDER BY number LIMIT 5`, function `sipHash64` would be evaluated after `ORDER BY` and `LIMIT`, which gives ~20x speed up. [#35623](https://github.com/ClickHouse/ClickHouse/pull/35623) ([Nikita Taranov](https://github.com/nickitat)).
* Sizes of hash tables used during aggregation now collected and used in later queries to avoid hash tables resizes. [#33439](https://github.com/ClickHouse/ClickHouse/pull/33439) ([Nikita Taranov](https://github.com/nickitat)).
* Improvement for hasAll function using SIMD instructions (SSE and AVX2). [#27653](https://github.com/ClickHouse/ClickHouse/pull/27653) ([youennL-cs](https://github.com/youennL-cs)). [#35723](https://github.com/ClickHouse/ClickHouse/pull/35723) ([Maksim Kita](https://github.com/kitaisreal)).
* Multiple changes to improve ASOF JOIN performance (1.2 - 1.6x as fast). It also adds support to use big integers. [#34733](https://github.com/ClickHouse/ClickHouse/pull/34733) ([Raúl Marín](https://github.com/Algunenano)).
* Improve performance of ASOF JOIN if key is native integer. [#35525](https://github.com/ClickHouse/ClickHouse/pull/35525) ([Maksim Kita](https://github.com/kitaisreal)).
* Parallelization of multipart upload into S3 storage. [#35343](https://github.com/ClickHouse/ClickHouse/pull/35343) ([Sergei Trifonov](https://github.com/serxa)).
* URL storage engine now downloads multiple chunks in parallel if the endpoint supports HTTP Range. Two additional settings were added, `max_download_threads` and `max_download_buffer_size`, which control maximum number of threads a single query can use to download the file and the maximum number of bytes each thread can process. [#35150](https://github.com/ClickHouse/ClickHouse/pull/35150) ([Antonio Andelic](https://github.com/antonio2368)).
* Use multiple threads to download objects from S3. Downloading is controllable using `max_download_threads` and `max_download_buffer_size` settings. [#35571](https://github.com/ClickHouse/ClickHouse/pull/35571) ([Antonio Andelic](https://github.com/antonio2368)).
* Narrow mutex scope when interacting with HDFS. Related to [#35292](https://github.com/ClickHouse/ClickHouse/issues/35292). [#35646](https://github.com/ClickHouse/ClickHouse/pull/35646) ([shuchaome](https://github.com/shuchaome)).
* Require mutations for per-table TTL only when it had been changed. [#35953](https://github.com/ClickHouse/ClickHouse/pull/35953) ([Azat Khuzhin](https://github.com/azat)).
#### Improvement
* Multiple improvements for schema inference. Use some tweaks and heuristics to determine numbers, strings, arrays, tuples and maps in CSV, TSV and TSVRaw data formats. Add setting `input_format_csv_use_best_effort_in_schema_inference` for CSV format that enables/disables using these heuristics, if it's disabled, we treat everything as string. Add similar setting `input_format_tsv_use_best_effort_in_schema_inference` for TSV/TSVRaw format. These settings are enabled by default. - Add Maps support for schema inference in Values format. - Fix possible segfault in schema inference in Values format. - Allow to skip columns with unsupported types in Arrow/ORC/Parquet formats. Add corresponding settings for it: `input_format_{parquet|orc|arrow}_skip_columns_with_unsupported_types_in_schema_inference`. These settings are disabled by default. - Allow to convert a column with type Null to a Nullable column with all NULL values in Arrow/Parquet formats. - Allow to specify column names in schema inference via setting `column_names_for_schema_inference` for formats that don't contain column names (like CSV, TSV, JSONCompactEachRow, etc) - Fix schema inference in ORC/Arrow/Parquet formats in terms of working with Nullable columns. Previously all inferred types were not Nullable and it blocked reading Nullable columns from data, now it's fixed and all inferred types are always Nullable (because we cannot understand that column is Nullable or not by reading the schema). - Fix schema inference in Template format with CSV escaping rules. [#35582](https://github.com/ClickHouse/ClickHouse/pull/35582) ([Kruglov Pavel](https://github.com/Avogar)).
* Add parallel parsing and schema inference for format `JSONAsObject`. [#35592](https://github.com/ClickHouse/ClickHouse/pull/35592) ([Anton Popov](https://github.com/CurtizJ)).
* Added a support for automatic schema inference to `s3Cluster` table function. Synced the signatures of `s3 ` and `s3Cluster`. [#35544](https://github.com/ClickHouse/ClickHouse/pull/35544) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
* Added support for schema inference for `hdfsCluster`. [#35602](https://github.com/ClickHouse/ClickHouse/pull/35602) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
* Add new setting `input_format_json_read_bools_as_numbers` that allows to infer and parse bools as numbers in JSON input formats. It's enabled by default. Suggested by @alexey-milovidov. [#35735](https://github.com/ClickHouse/ClickHouse/pull/35735) ([Kruglov Pavel](https://github.com/Avogar)).
* Improve columns ordering in schema inference for formats TSKV and JSONEachRow, closes [#35640](https://github.com/ClickHouse/ClickHouse/issues/35640). Don't stop schema inference when reading empty row in schema inference for formats TSKV and JSONEachRow. [#35724](https://github.com/ClickHouse/ClickHouse/pull/35724) ([Kruglov Pavel](https://github.com/Avogar)).
* Add settings `input_format_orc_case_insensitive_column_matching`, `input_format_arrow_case_insensitive_column_matching`, and `input_format_parquet_case_insensitive_column_matching` which allows ClickHouse to use case insensitive matching of columns while reading data from ORC, Arrow or Parquet files. [#35459](https://github.com/ClickHouse/ClickHouse/pull/35459) ([Antonio Andelic](https://github.com/antonio2368)).
* Added `is_secure` column to `system.query_log` which denotes if the client is using a secure connection over TCP or HTTP. [#35705](https://github.com/ClickHouse/ClickHouse/pull/35705) ([Antonio Andelic](https://github.com/antonio2368)).
* Now `kafka_num_consumers` can be bigger than amount of physical cores in case of low resource machine (less than 16 cores). [#35926](https://github.com/ClickHouse/ClickHouse/pull/35926) ([alesapin](https://github.com/alesapin)).
* Add some basic metrics to monitor engine=Kafka tables. [#35916](https://github.com/ClickHouse/ClickHouse/pull/35916) ([filimonov](https://github.com/filimonov)).
* Now it's not allowed to `ALTER TABLE ... RESET SETTING` for non-existing settings for MergeTree engines family. Fixes [#35816](https://github.com/ClickHouse/ClickHouse/issues/35816). [#35884](https://github.com/ClickHouse/ClickHouse/pull/35884) ([alesapin](https://github.com/alesapin)).
* Now some `ALTER MODIFY COLUMN` queries for `Arrays` and `Nullable` types can be done at metadata level without mutations. For example, alter from `Array(Enum8('Option1'=1))` to `Array(Enum8('Option1'=1, 'Option2'=2))`. [#35882](https://github.com/ClickHouse/ClickHouse/pull/35882) ([alesapin](https://github.com/alesapin)).
* Added an animation to the hourglass icon to indicate to the user that a query is running. [#35860](https://github.com/ClickHouse/ClickHouse/pull/35860) ([peledni](https://github.com/peledni)).
* support ALTER TABLE t DETACH PARTITION (ALL). [#35794](https://github.com/ClickHouse/ClickHouse/pull/35794) ([awakeljw](https://github.com/awakeljw)).
* Improve projection analysis to optimize trivial queries such as `count()`. [#35788](https://github.com/ClickHouse/ClickHouse/pull/35788) ([Amos Bird](https://github.com/amosbird)).
* Support schema inference for insert select with using `input` table function. Get schema from insertion table instead of inferring it from the data in case of insert select from table functions that support schema inference. Closes [#35639](https://github.com/ClickHouse/ClickHouse/issues/35639). [#35760](https://github.com/ClickHouse/ClickHouse/pull/35760) ([Kruglov Pavel](https://github.com/Avogar)).
* Respect `remote_url_allow_hosts` for Hive tables. [#35743](https://github.com/ClickHouse/ClickHouse/pull/35743) ([李扬](https://github.com/taiyang-li)).
* Implement `send_logs_level` for clickhouse-local. Closes [#35653](https://github.com/ClickHouse/ClickHouse/issues/35653). [#35716](https://github.com/ClickHouse/ClickHouse/pull/35716) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Closes [#35641](https://github.com/ClickHouse/ClickHouse/issues/35641) Allow `EPHEMERAL` columns without explicit default expression. [#35706](https://github.com/ClickHouse/ClickHouse/pull/35706) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Add profile event counter `AsyncInsertBytes` about size of async INSERTs. [#35644](https://github.com/ClickHouse/ClickHouse/pull/35644) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Improve the pipeline description for JOIN. [#35612](https://github.com/ClickHouse/ClickHouse/pull/35612) ([何李夫](https://github.com/helifu)).
* Deduce absolute hdfs config path. [#35572](https://github.com/ClickHouse/ClickHouse/pull/35572) ([李扬](https://github.com/taiyang-li)).
* Improve pasting performance and compatibility of clickhouse-client. This helps [#35501](https://github.com/ClickHouse/ClickHouse/issues/35501). [#35541](https://github.com/ClickHouse/ClickHouse/pull/35541) ([Amos Bird](https://github.com/amosbird)).
* It was possible to get stack overflow in distributed queries if one of the settings `async_socket_for_remote` and `use_hedged_requests` is enabled while parsing very deeply nested data type (at least in debug build). Closes [#35509](https://github.com/ClickHouse/ClickHouse/issues/35509). [#35524](https://github.com/ClickHouse/ClickHouse/pull/35524) ([Kruglov Pavel](https://github.com/Avogar)).
* Add sizes of subcolumns to `system.parts_columns` table. [#35488](https://github.com/ClickHouse/ClickHouse/pull/35488) ([Anton Popov](https://github.com/CurtizJ)).
* Add explicit table info to the scan node of query plan and pipeline. [#35460](https://github.com/ClickHouse/ClickHouse/pull/35460) ([何李夫](https://github.com/helifu)).
* Allow server to bind to low-numbered ports (e.g. 443). ClickHouse installation script will set `cap_net_bind_service` to the binary file. [#35451](https://github.com/ClickHouse/ClickHouse/pull/35451) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Fix INSERT INTO table FROM INFILE: it did not display the progress bar. [#35429](https://github.com/ClickHouse/ClickHouse/pull/35429) ([xiedeyantu](https://github.com/xiedeyantu)).
* Add arguments `--user`, `--password`, `--host`, `--port` for `clickhouse-diagnostics` tool. [#35422](https://github.com/ClickHouse/ClickHouse/pull/35422) ([李扬](https://github.com/taiyang-li)).
* Support uuid for Postgres engines. Closes [#35384](https://github.com/ClickHouse/ClickHouse/issues/35384). [#35403](https://github.com/ClickHouse/ClickHouse/pull/35403) ([Kseniia Sumarokova](https://github.com/kssenii)).
* For table function `s3cluster` or `HDFSCluster` or `hive`, we can't get right `AccessType` by `StorageFactory::instance().getSourceAccessType(getStorageTypeName())`. This pr fix it. [#35365](https://github.com/ClickHouse/ClickHouse/pull/35365) ([李扬](https://github.com/taiyang-li)).
* Remove `--testmode` option for clickhouse-client, enable it unconditionally. [#35354](https://github.com/ClickHouse/ClickHouse/pull/35354) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Don't allow `wchc` operation (four letter command) for clickhouse-keeper. [#35320](https://github.com/ClickHouse/ClickHouse/pull/35320) ([zhangyuli1](https://github.com/zhangyuli1)).
* Add function `getTypeSerializationStreams`. For a specified type (which is detected from column), it returns an array with all the serialization substream paths. This function is useful mainly for developers. [#35290](https://github.com/ClickHouse/ClickHouse/pull/35290) ([李扬](https://github.com/taiyang-li)).
* If `port` is not specified in cluster configuration, default server port will be used. This closes [#34769](https://github.com/ClickHouse/ClickHouse/issues/34769). [#34772](https://github.com/ClickHouse/ClickHouse/pull/34772) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Use `minmax` index for orc/parquet file in Hive Engine. Related PR: https://github.com/ClickHouse/arrow/pull/10. [#34631](https://github.com/ClickHouse/ClickHouse/pull/34631) ([李扬](https://github.com/taiyang-li)).
* System log tables now allow to specify COMMENT in ENGINE declaration. Closes [#33768](https://github.com/ClickHouse/ClickHouse/issues/33768). [#34536](https://github.com/ClickHouse/ClickHouse/pull/34536) ([Maksim Kita](https://github.com/kitaisreal)).
* Proper support of setting `max_rows_to_read` in case of reading in order of sorting key and specified limit. Previously the exception `Limit for rows or bytes to read exceeded` could be thrown even if query actually requires to read less amount of rows. [#33230](https://github.com/ClickHouse/ClickHouse/pull/33230) ([Anton Popov](https://github.com/CurtizJ)).
* Respect only quota & period from cgroups, ignore shares (which are not really limit the number of the cores which can be used). [#35815](https://github.com/ClickHouse/ClickHouse/pull/35815) ([filimonov](https://github.com/filimonov)).
#### Build/Testing/Packaging Improvement
* Add next batch of randomization settings in functional tests. [#35047](https://github.com/ClickHouse/ClickHouse/pull/35047) ([Kruglov Pavel](https://github.com/Avogar)).
* Add backward compatibility check in stress test. Closes [#25088](https://github.com/ClickHouse/ClickHouse/issues/25088). [#27928](https://github.com/ClickHouse/ClickHouse/pull/27928) ([Kruglov Pavel](https://github.com/Avogar)).
* Migrate package building to `nfpm` - Deprecate `release` script in favor of `packages/build` - Build everything in clickhouse/binary-builder image (cleanup: clickhouse/deb-builder) - Add symbol stripping to cmake (todo: use $prefix/lib/$bin_dir/clickhouse/$binary.debug) - Fix issue with DWARF symbols - Add Alpine APK packages - Rename `alien` to `additional_pkgs`. [#33664](https://github.com/ClickHouse/ClickHouse/pull/33664) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Add a night scan and upload for Coverity. [#34895](https://github.com/ClickHouse/ClickHouse/pull/34895) ([Boris Kuschel](https://github.com/bkuschel)).
* A dedicated small package for `clickhouse-keeper`. [#35308](https://github.com/ClickHouse/ClickHouse/pull/35308) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Running with podman was failing: it complains about specifying the same volume twice. [#35978](https://github.com/ClickHouse/ClickHouse/pull/35978) ([Roman Nikonov](https://github.com/nic11)).
* Minor improvement in contrib/krb5 build configuration. [#35832](https://github.com/ClickHouse/ClickHouse/pull/35832) ([Anton Kozlov](https://github.com/tonickkozlov)).
* Add a label to recognize a building task for every image. [#35583](https://github.com/ClickHouse/ClickHouse/pull/35583) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Apply `black` formatter to python code and add a per-commit check. [#35466](https://github.com/ClickHouse/ClickHouse/pull/35466) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Redo alpine image to use clean Dockerfile. Create a script in tests/ci to build both ubuntu and alpine images. Add clickhouse-keeper image (cc @nikitamikhaylov). Add build check to PullRequestCI. Add a job to a ReleaseCI. Add a job to MasterCI to build and push `clickhouse/clickhouse-server:head` and `clickhouse/clickhouse-keeper:head` images for each merged PR. [#35211](https://github.com/ClickHouse/ClickHouse/pull/35211) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Fix stress-test report in CI, now we upload the runlog with information about started stress tests only once. [#35093](https://github.com/ClickHouse/ClickHouse/pull/35093) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Switch to libcxx / libcxxabi from LLVM 14. [#34906](https://github.com/ClickHouse/ClickHouse/pull/34906) ([Raúl Marín](https://github.com/Algunenano)).
* Update unixodbc to mitigate CVE-2018-7485. Note: this CVE is not relevant for ClickHouse as it implements its own isolation layer for ODBC. [#35943](https://github.com/ClickHouse/ClickHouse/pull/35943) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### Bug Fix
* Added settings `input_format_ipv4_default_on_conversion_error`, `input_format_ipv6_default_on_conversion_error` to allow insert of invalid ip address values as default into tables. Closes [#35726](https://github.com/ClickHouse/ClickHouse/issues/35726). [#35733](https://github.com/ClickHouse/ClickHouse/pull/35733) ([Maksim Kita](https://github.com/kitaisreal)).
* Avoid erasing columns from a block if it doesn't exist while reading data from Hive. [#35393](https://github.com/ClickHouse/ClickHouse/pull/35393) ([lgbo](https://github.com/lgbo-ustc)).
* Add type checking when creating materialized view. Close: [#23684](https://github.com/ClickHouse/ClickHouse/issues/23684). [#24896](https://github.com/ClickHouse/ClickHouse/pull/24896) ([hexiaoting](https://github.com/hexiaoting)).
* Fix formatting of INSERT INFILE queries (missing quotes). [#35886](https://github.com/ClickHouse/ClickHouse/pull/35886) ([Azat Khuzhin](https://github.com/azat)).
* Disable `session_log` because memory safety issue has been found by fuzzing. See [#35714](https://github.com/ClickHouse/ClickHouse/issues/35714). [#35873](https://github.com/ClickHouse/ClickHouse/pull/35873) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Avoid processing per-column TTL multiple times. [#35820](https://github.com/ClickHouse/ClickHouse/pull/35820) ([Azat Khuzhin](https://github.com/azat)).
* Fix inserts to columns of type `Object` in case when there is data related to several partitions in insert query. [#35806](https://github.com/ClickHouse/ClickHouse/pull/35806) ([Anton Popov](https://github.com/CurtizJ)).
* Fix bug in indexes of not presented columns in -WithNames formats that led to error `INCORRECT_NUMBER_OF_COLUMNS ` when the number of columns is more than 256. Closes [#35793](https://github.com/ClickHouse/ClickHouse/issues/35793). [#35803](https://github.com/ClickHouse/ClickHouse/pull/35803) ([Kruglov Pavel](https://github.com/Avogar)).
* Fixes [#35751](https://github.com/ClickHouse/ClickHouse/issues/35751). [#35799](https://github.com/ClickHouse/ClickHouse/pull/35799) ([Nikolay Degterinsky](https://github.com/evillique)).
* Fix for reading from HDFS in Snappy format. [#35771](https://github.com/ClickHouse/ClickHouse/pull/35771) ([shuchaome](https://github.com/shuchaome)).
* Fix bug in conversion from custom types to string that could lead to segfault or unexpected error messages. Closes [#35752](https://github.com/ClickHouse/ClickHouse/issues/35752). [#35755](https://github.com/ClickHouse/ClickHouse/pull/35755) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix any/all (subquery) implementation. Closes [#35489](https://github.com/ClickHouse/ClickHouse/issues/35489). [#35727](https://github.com/ClickHouse/ClickHouse/pull/35727) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix dropping non-empty database in clickhouse-local. Closes [#35692](https://github.com/ClickHouse/ClickHouse/issues/35692). [#35711](https://github.com/ClickHouse/ClickHouse/pull/35711) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix bug in creating materialized view with subquery after server restart. Materialized view was not getting updated after inserts into underlying table after server restart. Closes [#35511](https://github.com/ClickHouse/ClickHouse/issues/35511). [#35691](https://github.com/ClickHouse/ClickHouse/pull/35691) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix possible `Can't adjust last granule` exception while reading subcolumns of experimental type `Object`. [#35687](https://github.com/ClickHouse/ClickHouse/pull/35687) ([Anton Popov](https://github.com/CurtizJ)).
* Enable build with JIT compilation by default. [#35683](https://github.com/ClickHouse/ClickHouse/pull/35683) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix possible loss of subcolumns in experimental type `Object`. [#35682](https://github.com/ClickHouse/ClickHouse/pull/35682) ([Anton Popov](https://github.com/CurtizJ)).
* Fix check ASOF JOIN key nullability, close [#35565](https://github.com/ClickHouse/ClickHouse/issues/35565). [#35674](https://github.com/ClickHouse/ClickHouse/pull/35674) ([Vladimir C](https://github.com/vdimir)).
* Fix part checking logic for parts with projections. Error happened when projection and main part had different types. This is similar to https://github.com/ClickHouse/ClickHouse/pull/33774 . The bug is addressed by @caoyang10. [#35667](https://github.com/ClickHouse/ClickHouse/pull/35667) ([Amos Bird](https://github.com/amosbird)).
* Fix server crash when large number of arguments are passed into `format` function. Please refer to the test file and see how to reproduce the crash. [#35651](https://github.com/ClickHouse/ClickHouse/pull/35651) ([Amos Bird](https://github.com/amosbird)).
* Fix usage of quotas with asynchronous inserts. [#35645](https://github.com/ClickHouse/ClickHouse/pull/35645) ([Anton Popov](https://github.com/CurtizJ)).
* Fix positional arguments with aliases. Closes [#35600](https://github.com/ClickHouse/ClickHouse/issues/35600). [#35620](https://github.com/ClickHouse/ClickHouse/pull/35620) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Check `remote_url_allow_hosts` before schema inference in URL engine Closes [#35064](https://github.com/ClickHouse/ClickHouse/issues/35064). [#35619](https://github.com/ClickHouse/ClickHouse/pull/35619) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix `HashJoin` when columns with `LowCardinality` type are used. This closes [#35548](https://github.com/ClickHouse/ClickHouse/issues/35548). [#35616](https://github.com/ClickHouse/ClickHouse/pull/35616) ([Antonio Andelic](https://github.com/antonio2368)).
* Fix possible segfault in MaterializedPostgreSQL which happened if exception occurred when data, collected in memory, was synced into underlying tables. Closes [#35611](https://github.com/ClickHouse/ClickHouse/issues/35611). [#35614](https://github.com/ClickHouse/ClickHouse/pull/35614) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Setting `database_atomic_wait_for_drop_and_detach_synchronously` worked incorrectly for `ATTACH TABLE` query when previously detached table is still in use, It's fixed. [#35594](https://github.com/ClickHouse/ClickHouse/pull/35594) ([tavplubix](https://github.com/tavplubix)).
* Fix HTTP headers with named collections, add compression_method. Closes [#35273](https://github.com/ClickHouse/ClickHouse/issues/35273). Closes [#35269](https://github.com/ClickHouse/ClickHouse/issues/35269). [#35593](https://github.com/ClickHouse/ClickHouse/pull/35593) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix s3 engine getting virtual columns. Closes [#35411](https://github.com/ClickHouse/ClickHouse/issues/35411). [#35586](https://github.com/ClickHouse/ClickHouse/pull/35586) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fixed return type deduction for `caseWithExpression`. The type of the ELSE branch is now correctly taken into account. [#35576](https://github.com/ClickHouse/ClickHouse/pull/35576) ([Antonio Andelic](https://github.com/antonio2368)).
* Fix parsing of IPv6 addresses longer than 39 characters. Closes [#34022](https://github.com/ClickHouse/ClickHouse/issues/34022). [#35539](https://github.com/ClickHouse/ClickHouse/pull/35539) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix cast into IPv4, IPv6 address in IN section. Fixes [#35528](https://github.com/ClickHouse/ClickHouse/issues/35528). [#35534](https://github.com/ClickHouse/ClickHouse/pull/35534) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix crash during short circuit function evaluation when one of arguments is nullable constant. Closes [#35497](https://github.com/ClickHouse/ClickHouse/issues/35497). Closes [#35496](https://github.com/ClickHouse/ClickHouse/issues/35496). [#35502](https://github.com/ClickHouse/ClickHouse/pull/35502) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix crash for function `throwIf` with constant arguments. [#35500](https://github.com/ClickHouse/ClickHouse/pull/35500) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix bug in Keeper which can lead to unstable client connections. Introduced in [#35031](https://github.com/ClickHouse/ClickHouse/issues/35031). [#35498](https://github.com/ClickHouse/ClickHouse/pull/35498) ([alesapin](https://github.com/alesapin)).
* Fix bug in function `if` when resulting column type differs with resulting data type that led to logical errors like `Logical error: 'Bad cast from type DB::ColumnVector<int> to DB::ColumnVector<long>'.`. Closes [#35367](https://github.com/ClickHouse/ClickHouse/issues/35367). [#35476](https://github.com/ClickHouse/ClickHouse/pull/35476) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix excessive logging when using S3 as backend for MergeTree or as separate table engine/function. Fixes [#30559](https://github.com/ClickHouse/ClickHouse/issues/30559). [#35434](https://github.com/ClickHouse/ClickHouse/pull/35434) ([alesapin](https://github.com/alesapin)).
* Now merges executed with zero copy replication (experimental) will not spam logs with message `Found parts with the same min block and with the same max block as the missing part _ on replica _. Hoping that it will eventually appear as a result of a merge.`. [#35430](https://github.com/ClickHouse/ClickHouse/pull/35430) ([alesapin](https://github.com/alesapin)).
* Skip possible exception if empty chunks appear in GroupingAggregatedTransform. [#35417](https://github.com/ClickHouse/ClickHouse/pull/35417) ([Nikita Taranov](https://github.com/nickitat)).
* Fix working with columns that are not needed in query in Arrow/Parquet/ORC formats, it prevents possible errors like `Unsupported <format> type <type> of an input column <column_name>` when file contains column with unsupported type and we don't use it in query. [#35406](https://github.com/ClickHouse/ClickHouse/pull/35406) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix for local cache for remote filesystem (experimental feature) for high concurrency on corner cases. [#35381](https://github.com/ClickHouse/ClickHouse/pull/35381) ([Kseniia Sumarokova](https://github.com/kssenii)). Fix possible deadlock in cache. [#35378](https://github.com/ClickHouse/ClickHouse/pull/35378) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix partition pruning in case of comparison with constant in `WHERE`. If column and constant had different types, overflow was possible. Query could return an incorrect empty result. This fixes [#35304](https://github.com/ClickHouse/ClickHouse/issues/35304). [#35334](https://github.com/ClickHouse/ClickHouse/pull/35334) ([Amos Bird](https://github.com/amosbird)).
* Fix schema inference for TSKV format while using small max_read_buffer_size. [#35332](https://github.com/ClickHouse/ClickHouse/pull/35332) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix mutations in tables with enabled sparse columns. [#35284](https://github.com/ClickHouse/ClickHouse/pull/35284) ([Anton Popov](https://github.com/CurtizJ)).
* Do not delay final part writing by default (fixes possible `Memory limit exceeded` during `INSERT` by adding `max_insert_delayed_streams_for_parallel_write` with default to 1000 for writes to s3 and disabled as before otherwise). [#34780](https://github.com/ClickHouse/ClickHouse/pull/34780) ([Azat Khuzhin](https://github.com/azat)).
## <a id="223"></a> ClickHouse release v22.3-lts, 2022-03-17

2
docker/packager/binary/build.sh
View File

@ -8,6 +8,8 @@ cache_status () {
ccache --show-stats ||:
}
git config --global --add safe.directory /build
mkdir -p build/cmake/toolchain/darwin-x86_64
tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64

2
docker/test/fasttest/run.sh
View File

@ -179,7 +179,7 @@ function clone_submodules
)
git submodule sync
git submodule update --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}"
git submodule update --jobs=16 --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}"
git submodule foreach git reset --hard
git submodule foreach git checkout @ -f
git submodule foreach git clean -xfd

2
docker/test/stateless/run.sh
View File

@ -92,6 +92,8 @@ function run_tests()
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
ADDITIONAL_OPTIONS+=('--replicated-database')
# Cannot be used with replicated database, due to distributed_ddl_output_mode=none
ADDITIONAL_OPTIONS+=('--no-left-queries-check')
ADDITIONAL_OPTIONS+=('--jobs')
ADDITIONAL_OPTIONS+=('2')
else

1
programs/server/Server.cpp
View File

@ -1339,7 +1339,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
global_context->getMergeTreeSettings().sanityCheck(settings);
global_context->getReplicatedMergeTreeSettings().sanityCheck(settings);
/// try set up encryption. There are some errors in config, error will be printed and server wouldn't start.
CompressionCodecEncrypted::Configuration::instance().load(config(), "encryption_codecs");

14
src/Client/ClientBase.cpp
View File

@ -1326,9 +1326,13 @@ bool ClientBase::receiveEndOfQuery()
onProgress(packet.progress);
break;
case Protocol::Server::ProfileEvents:
onProfileEvents(packet.block);
break;
default:
throw NetException(
"Unexpected packet from server (expected Exception, EndOfStream or Log, got "
"Unexpected packet from server (expected Exception, EndOfStream, Log, Progress or ProfileEvents. Got "
+ String(Protocol::Server::toString(packet.type)) + ")",
ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
}
@ -1476,7 +1480,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
const char *& this_query_begin, const char *& this_query_end, const char * all_queries_end,
String & query_to_execute, ASTPtr & parsed_query, const String & all_queries_text,
std::optional<Exception> & current_exception)
std::unique_ptr<Exception> & current_exception)
{
if (!is_interactive && cancelled)
return MultiQueryProcessingStage::QUERIES_END;
@ -1514,7 +1518,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
}
catch (Exception & e)
{
current_exception.emplace(e);
current_exception.reset(e.clone());
return MultiQueryProcessingStage::PARSING_EXCEPTION;
}
@ -1595,7 +1599,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
String full_query; // full_query is the query + inline INSERT data + trailing comments (the latter is our best guess for now).
String query_to_execute;
ASTPtr parsed_query;
std::optional<Exception> current_exception;
std::unique_ptr<Exception> current_exception;
while (true)
{
@ -1939,7 +1943,7 @@ void ClientBase::runInteractive()
{
/// We don't need to handle the test hints in the interactive mode.
std::cerr << "Exception on client:" << std::endl << getExceptionMessage(e, print_stack_trace, true) << std::endl << std::endl;
client_exception = std::make_unique<Exception>(e);
client_exception.reset(e.clone());
}
if (client_exception)

2
src/Client/ClientBase.h
View File

@ -80,7 +80,7 @@ protected:
MultiQueryProcessingStage analyzeMultiQueryText(
const char *& this_query_begin, const char *& this_query_end, const char * all_queries_end,
String & query_to_execute, ASTPtr & parsed_query, const String & all_queries_text,
std::optional<Exception> & current_exception);
std::unique_ptr<Exception> & current_exception);
static void clearTerminal();
void showClientVersion();

14
src/Client/LocalConnection.cpp
View File

@ -154,17 +154,17 @@ void LocalConnection::sendQuery(
catch (const Exception & e)
{
state->io.onException();
state->exception.emplace(e);
state->exception.reset(e.clone());
}
catch (const std::exception & e)
{
state->io.onException();
state->exception.emplace(Exception::CreateFromSTDTag{}, e);
state->exception = std::make_unique<Exception>(Exception::CreateFromSTDTag{}, e);
}
catch (...)
{
state->io.onException();
state->exception.emplace("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
state->exception = std::make_unique<Exception>("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
}
}
@ -260,17 +260,17 @@ bool LocalConnection::poll(size_t)
catch (const Exception & e)
{
state->io.onException();
state->exception.emplace(e);
state->exception.reset(e.clone());
}
catch (const std::exception & e)
{
state->io.onException();
state->exception.emplace(Exception::CreateFromSTDTag{}, e);
state->exception = std::make_unique<Exception>(Exception::CreateFromSTDTag{}, e);
}
catch (...)
{
state->io.onException();
state->exception.emplace("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
state->exception = std::make_unique<Exception>("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
}
}
@ -434,7 +434,7 @@ Packet LocalConnection::receivePacket()
}
case Protocol::Server::Exception:
{
packet.exception = std::make_unique<Exception>(*state->exception);
packet.exception.reset(state->exception->clone());
next_packet_type.reset();
break;
}

2
src/Client/LocalConnection.h
View File

@ -32,7 +32,7 @@ struct LocalQueryState
std::unique_ptr<PushingAsyncPipelineExecutor> pushing_async_executor;
InternalProfileEventsQueuePtr profile_queue;
std::optional<Exception> exception;
std::unique_ptr<Exception> exception;
/// Current block to be sent next.
std::optional<Block> block;

23
src/Common/Exception.cpp
View File

@ -555,13 +555,24 @@ std::string ParsingException::displayText() const
{
try
{
if (line_number == -1)
formatted_message = message();
else
formatted_message = message() + fmt::format(": (at row {})\n", line_number);
formatted_message = message();
bool need_newline = false;
if (!file_name.empty())
{
formatted_message += fmt::format(": (in file/uri {})", file_name);
need_newline = true;
}
if (line_number != -1)
{
formatted_message += fmt::format(": (at row {})", line_number);
need_newline = true;
}
if (need_newline)
formatted_message += "\n";
}
catch (...)
{}
catch (...) {}
if (!formatted_message.empty())
{

7
src/Common/Exception.h
View File

@ -132,8 +132,15 @@ public:
int getLineNumber() const { return line_number; }
void setLineNumber(int line_number_) { line_number = line_number_;}
const String getFileName() const { return file_name; }
void setFileName(const String & file_name_) { file_name = file_name_; }
Exception * clone() const override { return new ParsingException(*this); }
void rethrow() const override { throw *this; }
private:
ssize_t line_number{-1};
String file_name;
mutable std::string formatted_message;
const char * name() const throw() override { return "DB::ParsingException"; }

7
src/DataTypes/DataTypeFactory.cpp
View File

@ -33,7 +33,12 @@ DataTypePtr DataTypeFactory::get(const String & full_name) const
/// Value 315 is known to cause stack overflow in some test configurations (debug build, sanitizers)
/// let's make the threshold significantly lower.
/// It is impractical for user to have complex data types with this depth.
static constexpr size_t data_type_max_parse_depth = 200;
#if defined(SANITIZER) || !defined(NDEBUG)
static constexpr size_t data_type_max_parse_depth = 150;
#else
static constexpr size_t data_type_max_parse_depth = 300;
#endif
ParserDataType parser;
ASTPtr ast = parseQuery(parser, full_name.data(), full_name.data() + full_name.size(), "data type", 0, data_type_max_parse_depth);

16
src/Dictionaries/getDictionaryConfigurationFromAST.cpp
View File

@ -4,6 +4,7 @@
#include <Poco/DOM/Document.h>
#include <Poco/DOM/Element.h>
#include <Poco/DOM/Text.h>
#include <Poco/Net/NetException.h>
#include <Poco/Util/XMLConfiguration.h>
#include <IO/WriteHelpers.h>
#include <Parsers/queryToString.h>
@ -629,11 +630,18 @@ getInfoIfClickHouseDictionarySource(DictionaryConfigurationPtr & config, Context
info.table_name = {database, table};
UInt16 default_port = secure ? global_context->getTCPPortSecure().value_or(0) : global_context->getTCPPort();
if (!isLocalAddress({host, port}, default_port))
return info;
try
{
UInt16 default_port = secure ? global_context->getTCPPortSecure().value_or(0) : global_context->getTCPPort();
if (isLocalAddress({host, port}, default_port))
info.is_local = true;
}
catch (const Poco::Net::DNSException &)
{
/// Server may fail to start if we cannot resolve some hostname. It's ok to ignore exception and leave is_local false.
tryLogCurrentException(__PRETTY_FUNCTION__);
}
info.is_local = true;
return info;
}

15
src/IO/BrotliReadBuffer.cpp
View File

@ -32,14 +32,13 @@ public:
};
BrotliReadBuffer::BrotliReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char *existing_memory, size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment)
, in(std::move(in_))
, brotli(std::make_unique<BrotliStateWrapper>())
, in_available(0)
, in_data(nullptr)
, out_capacity(0)
, out_data(nullptr)
, eof_flag(false)
: CompressedReadBufferWrapper(std::move(in_), buf_size, existing_memory, alignment)
, brotli(std::make_unique<BrotliStateWrapper>())
, in_available(0)
, in_data(nullptr)
, out_capacity(0)
, out_data(nullptr)
, eof_flag(false)
{
}

6
src/IO/BrotliReadBuffer.h
View File

@ -1,13 +1,13 @@
#pragma once
#include <IO/ReadBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressedReadBufferWrapper.h>
namespace DB
{
class BrotliReadBuffer : public BufferWithOwnMemory<ReadBuffer>
class BrotliReadBuffer : public CompressedReadBufferWrapper
{
public:
explicit BrotliReadBuffer(
@ -21,8 +21,6 @@ public:
private:
bool nextImpl() override;
std::unique_ptr<ReadBuffer> in;
class BrotliStateWrapper;
std::unique_ptr<BrotliStateWrapper> brotli;

7
src/IO/Bzip2ReadBuffer.cpp
View File

@ -39,10 +39,9 @@ public:
};
Bzip2ReadBuffer::Bzip2ReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char *existing_memory, size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment)
, in(std::move(in_))
, bz(std::make_unique<Bzip2StateWrapper>())
, eof_flag(false)
: CompressedReadBufferWrapper(std::move(in_), buf_size, existing_memory, alignment)
, bz(std::make_unique<Bzip2StateWrapper>())
, eof_flag(false)
{
}

6
src/IO/Bzip2ReadBuffer.h
View File

@ -1,13 +1,13 @@
#pragma once
#include <IO/ReadBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressedReadBufferWrapper.h>
namespace DB
{
class Bzip2ReadBuffer : public BufferWithOwnMemory<ReadBuffer>
class Bzip2ReadBuffer : public CompressedReadBufferWrapper
{
public:
explicit Bzip2ReadBuffer(
@ -21,8 +21,6 @@ public:
private:
bool nextImpl() override;
std::unique_ptr<ReadBuffer> in;
class Bzip2StateWrapper;
std::unique_ptr<Bzip2StateWrapper> bz;

25
src/IO/CompressedReadBufferWrapper.h Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#include <IO/BufferWithOwnMemory.h>
#include <IO/ReadBuffer.h>
namespace DB
{
class CompressedReadBufferWrapper : public BufferWithOwnMemory<ReadBuffer>
{
public:
CompressedReadBufferWrapper(
std::unique_ptr<ReadBuffer> in_,
size_t buf_size,
char * existing_memory,
size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment)
, in(std::move(in_)) {}
const ReadBuffer & getWrappedReadBuffer() const { return *in; }
protected:
std::unique_ptr<ReadBuffer> in;
};
}

12
src/IO/CompressionMethod.cpp
View File

@ -98,8 +98,7 @@ CompressionMethod chooseCompressionMethod(const std::string & path, const std::s
ErrorCodes::NOT_IMPLEMENTED);
}
std::unique_ptr<ReadBuffer> wrapReadBufferWithCompressionMethod(
static std::unique_ptr<CompressedReadBufferWrapper> createCompressedWrapper(
std::unique_ptr<ReadBuffer> nested, CompressionMethod method, size_t buf_size, char * existing_memory, size_t alignment)
{
if (method == CompressionMethod::Gzip || method == CompressionMethod::Zlib)
@ -123,10 +122,15 @@ std::unique_ptr<ReadBuffer> wrapReadBufferWithCompressionMethod(
return std::make_unique<HadoopSnappyReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
#endif
throw Exception("Unsupported compression method", ErrorCodes::NOT_IMPLEMENTED);
}
std::unique_ptr<ReadBuffer> wrapReadBufferWithCompressionMethod(
std::unique_ptr<ReadBuffer> nested, CompressionMethod method, size_t buf_size, char * existing_memory, size_t alignment)
{
if (method == CompressionMethod::None)
return nested;
throw Exception("Unsupported compression method", ErrorCodes::NOT_IMPLEMENTED);
return createCompressedWrapper(std::move(nested), method, buf_size, existing_memory, alignment);
}

3
src/IO/HadoopSnappyReadBuffer.cpp
View File

@ -166,8 +166,7 @@ HadoopSnappyDecoder::Status HadoopSnappyDecoder::readBlock(size_t * avail_in, co
}
HadoopSnappyReadBuffer::HadoopSnappyReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment)
, in(std::move(in_))
: CompressedReadBufferWrapper(std::move(in_), buf_size, existing_memory, alignment)
, decoder(std::make_unique<HadoopSnappyDecoder>())
, in_available(0)
, in_data(nullptr)

5
src/IO/HadoopSnappyReadBuffer.h
View File

@ -6,7 +6,7 @@
#include <memory>
#include <IO/ReadBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressedReadBufferWrapper.h>
namespace DB
{
@ -67,7 +67,7 @@ private:
};
/// HadoopSnappyReadBuffer implements read buffer for data compressed with hadoop-snappy format.
class HadoopSnappyReadBuffer : public BufferWithOwnMemory<ReadBuffer>
class HadoopSnappyReadBuffer : public CompressedReadBufferWrapper
{
public:
using Status = HadoopSnappyDecoder::Status;
@ -99,7 +99,6 @@ public:
private:
bool nextImpl() override;
std::unique_ptr<ReadBuffer> in;
std::unique_ptr<HadoopSnappyDecoder> decoder;
size_t in_available;

2
src/IO/LZMAInflatingReadBuffer.cpp
View File

@ -8,7 +8,7 @@ namespace ErrorCodes
}
LZMAInflatingReadBuffer::LZMAInflatingReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment), in(std::move(in_)), eof_flag(false)
: CompressedReadBufferWrapper(std::move(in_), buf_size, existing_memory, alignment), eof_flag(false)
{
lstr = LZMA_STREAM_INIT;
lstr.allocator = nullptr;

6
src/IO/LZMAInflatingReadBuffer.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressedReadBufferWrapper.h>
#include <IO/ReadBuffer.h>
#include <lzma.h>
@ -8,7 +8,7 @@
namespace DB
{
class LZMAInflatingReadBuffer : public BufferWithOwnMemory<ReadBuffer>
class LZMAInflatingReadBuffer : public CompressedReadBufferWrapper
{
public:
explicit LZMAInflatingReadBuffer(
@ -22,9 +22,7 @@ public:
private:
bool nextImpl() override;
std::unique_ptr<ReadBuffer> in;
lzma_stream lstr;
bool eof_flag;
};

3
src/IO/Lz4InflatingReadBuffer.cpp
View File

@ -8,8 +8,7 @@ namespace ErrorCodes
}
Lz4InflatingReadBuffer::Lz4InflatingReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment)
, in(std::move(in_))
: CompressedReadBufferWrapper(std::move(in_), buf_size, existing_memory, alignment)
, in_data(nullptr)
, out_data(nullptr)
, in_available(0)

6
src/IO/Lz4InflatingReadBuffer.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressedReadBufferWrapper.h>
#include <IO/CompressionMethod.h>
#include <IO/ReadBuffer.h>
@ -11,7 +11,7 @@
namespace DB
{
class Lz4InflatingReadBuffer : public BufferWithOwnMemory<ReadBuffer>
class Lz4InflatingReadBuffer : public CompressedReadBufferWrapper
{
public:
explicit Lz4InflatingReadBuffer(
@ -25,8 +25,6 @@ public:
private:
bool nextImpl() override;
std::unique_ptr<ReadBuffer> in;
LZ4F_dctx* dctx;
void * in_data;

2
src/IO/ParallelReadBuffer.h
View File

@ -85,6 +85,8 @@ public:
std::optional<size_t> getTotalSize() override;
off_t getPosition() override;
const ReadBufferFactory & getReadBufferFactory() const { return *reader_factory; }
private:
/// Reader in progress with a list of read segments
struct ReadWorker

5
src/IO/ReadBufferFromAzureBlobStorage.h
View File

@ -7,12 +7,13 @@
#include <IO/HTTPCommon.h>
#include <IO/SeekableReadBuffer.h>
#include <IO/ReadSettings.h>
#include <IO/WithFileName.h>
#include <azure/storage/blobs.hpp>
namespace DB
{
class ReadBufferFromAzureBlobStorage : public SeekableReadBuffer
class ReadBufferFromAzureBlobStorage : public SeekableReadBuffer, public WithFileName
{
public:
@ -33,6 +34,8 @@ public:
size_t getFileOffsetOfBufferEnd() const override { return offset; }
String getFileName() const override { return path; }
private:
void initialize();

4
src/IO/ReadBufferFromFileBase.h
View File

@ -2,6 +2,7 @@
#include <IO/BufferWithOwnMemory.h>
#include <IO/SeekableReadBuffer.h>
#include <IO/WithFileName.h>
#include <base/time.h>
#include <functional>
@ -19,7 +20,7 @@
namespace DB
{
class ReadBufferFromFileBase : public BufferWithOwnMemory<SeekableReadBuffer>
class ReadBufferFromFileBase : public BufferWithOwnMemory<SeekableReadBuffer>, public WithFileName
{
public:
ReadBufferFromFileBase();
@ -29,7 +30,6 @@ public:
size_t alignment,
std::optional<size_t> file_size_ = std::nullopt);
~ReadBufferFromFileBase() override;
virtual std::string getFileName() const = 0;
/// It is possible to get information about the time of each reading.
struct ProfileInfo

9
src/IO/ReadBufferFromS3.h
View File

@ -12,6 +12,7 @@
#include <IO/ReadBuffer.h>
#include <IO/ReadSettings.h>
#include <IO/SeekableReadBuffer.h>
#include <IO/WithFileName.h>
#include <aws/s3/model/GetObjectResult.h>
@ -25,7 +26,7 @@ namespace DB
/**
* Perform S3 HTTP GET request and provide response to read.
*/
class ReadBufferFromS3 : public SeekableReadBufferWithSize
class ReadBufferFromS3 : public SeekableReadBufferWithSize, public WithFileName
{
private:
std::shared_ptr<Aws::S3::S3Client> client_ptr;
@ -70,6 +71,8 @@ public:
size_t getFileOffsetOfBufferEnd() const override { return offset; }
String getFileName() const override { return bucket + "/" + key; }
private:
std::unique_ptr<ReadBuffer> initialize();
@ -83,7 +86,7 @@ private:
};
/// Creates separate ReadBufferFromS3 for sequence of ranges of particular object
class ReadBufferS3Factory : public ParallelReadBuffer::ReadBufferFactory
class ReadBufferS3Factory : public ParallelReadBuffer::ReadBufferFactory, public WithFileName
{
public:
explicit ReadBufferS3Factory(
@ -113,6 +116,8 @@ public:
std::optional<size_t> getTotalSize() override;
String getFileName() const override { return bucket + "/" + key; }
private:
std::shared_ptr<Aws::S3::S3Client> client_ptr;
const String bucket;

9
src/IO/ReadWriteBufferFromHTTP.h
View File

@ -9,6 +9,7 @@
#include <IO/ReadBufferFromIStream.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadSettings.h>
#include <IO/WithFileName.h>
#include <base/logger_useful.h>
#include <base/sleep.h>
#include <base/types.h>
@ -85,7 +86,7 @@ public:
namespace detail
{
template <typename UpdatableSessionPtr>
class ReadWriteBufferFromHTTPBase : public SeekableReadBufferWithSize
class ReadWriteBufferFromHTTPBase : public SeekableReadBufferWithSize, public WithFileName
{
public:
using HTTPHeaderEntry = std::tuple<std::string, std::string>;
@ -223,6 +224,8 @@ namespace detail
return read_range.end;
}
String getFileName() const override { return uri.toString(); }
enum class InitializeError
{
/// If error is not retriable, `exception` variable must be set.
@ -674,7 +677,7 @@ public:
}
};
class RangedReadWriteBufferFromHTTPFactory : public ParallelReadBuffer::ReadBufferFactory
class RangedReadWriteBufferFromHTTPFactory : public ParallelReadBuffer::ReadBufferFactory, public WithFileName
{
using OutStreamCallback = ReadWriteBufferFromHTTP::OutStreamCallback;
@ -748,6 +751,8 @@ public:
std::optional<size_t> getTotalSize() override { return total_object_size; }
String getFileName() const override { return uri.toString(); }
private:
RangeGenerator range_generator;
size_t total_object_size;

26
src/IO/WithFileName.cpp Normal file
View File

@ -0,0 +1,26 @@
#include <IO/WithFileName.h>
#include <IO/CompressedReadBufferWrapper.h>
#include <IO/ParallelReadBuffer.h>
namespace DB
{
template <typename T>
static String getFileName(const T & entry)
{
if (const auto * with_file_name = dynamic_cast<const WithFileName *>(&entry))
return with_file_name->getFileName();
return "";
}
String getFileNameFromReadBuffer(const ReadBuffer & in)
{
if (const auto * compressed = dynamic_cast<const CompressedReadBufferWrapper *>(&in))
return getFileName(compressed->getWrappedReadBuffer());
else if (const auto * parallel = dynamic_cast<const ParallelReadBuffer *>(&in))
return getFileName(parallel->getReadBufferFactory());
else
return getFileName(in);
}
}

18
src/IO/WithFileName.h Normal file
View File

@ -0,0 +1,18 @@
#pragma once
#include <base/types.h>
namespace DB
{
class ReadBuffer;
class WithFileName
{
public:
virtual String getFileName() const = 0;
virtual ~WithFileName() = default;
};
String getFileNameFromReadBuffer(const ReadBuffer & in);
}

3
src/IO/ZlibInflatingReadBuffer.cpp
View File

@ -14,8 +14,7 @@ ZlibInflatingReadBuffer::ZlibInflatingReadBuffer(
size_t buf_size,
char * existing_memory,
size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment)
, in(std::move(in_))
: CompressedReadBufferWrapper(std::move(in_), buf_size, existing_memory, alignment)
, eof_flag(false)
{
zstr.zalloc = nullptr;

5
src/IO/ZlibInflatingReadBuffer.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <IO/ReadBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressedReadBufferWrapper.h>
#include <IO/CompressionMethod.h>
#include <zlib.h>
@ -16,7 +16,7 @@ namespace ErrorCodes
/// Reads compressed data from ReadBuffer in_ and performs decompression using zlib library.
/// This buffer is able to seamlessly decompress multiple concatenated zlib streams.
class ZlibInflatingReadBuffer : public BufferWithOwnMemory<ReadBuffer>
class ZlibInflatingReadBuffer : public CompressedReadBufferWrapper
{
public:
ZlibInflatingReadBuffer(
@ -31,7 +31,6 @@ public:
private:
bool nextImpl() override;
std::unique_ptr<ReadBuffer> in;
z_stream zstr;
bool eof_flag;
};

2
src/IO/ZstdInflatingReadBuffer.cpp
View File

@ -9,7 +9,7 @@ namespace ErrorCodes
}
ZstdInflatingReadBuffer::ZstdInflatingReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment), in(std::move(in_))
: CompressedReadBufferWrapper(std::move(in_), buf_size, existing_memory, alignment)
{
dctx = ZSTD_createDCtx();
input = {nullptr, 0, 0};

5
src/IO/ZstdInflatingReadBuffer.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressedReadBufferWrapper.h>
#include <IO/CompressionMethod.h>
#include <IO/ReadBuffer.h>
@ -13,7 +13,7 @@ namespace ErrorCodes
{
}
class ZstdInflatingReadBuffer : public BufferWithOwnMemory<ReadBuffer>
class ZstdInflatingReadBuffer : public CompressedReadBufferWrapper
{
public:
explicit ZstdInflatingReadBuffer(
@ -27,7 +27,6 @@ public:
private:
bool nextImpl() override;
std::unique_ptr<ReadBuffer> in;
ZSTD_DCtx * dctx;
ZSTD_inBuffer input;
ZSTD_outBuffer output;

52
src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
View File

@ -14,6 +14,7 @@
#include <Processors/QueryPlan/QueryPlan.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
#include <Processors/QueryPlan/DistributedCreateLocalPlan.h>
#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
@ -46,55 +47,6 @@ SelectStreamFactory::SelectStreamFactory(
{
}
namespace
{
ActionsDAGPtr getConvertingDAG(const Block & block, const Block & header)
{
/// Convert header structure to expected.
/// Also we ignore constants from result and replace it with constants from header.
/// It is needed for functions like `now64()` or `randConstant()` because their values may be different.
return ActionsDAG::makeConvertingActions(
block.getColumnsWithTypeAndName(),
header.getColumnsWithTypeAndName(),
ActionsDAG::MatchColumnsMode::Name,
true);
}
void addConvertingActions(QueryPlan & plan, const Block & header)
{
if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header))
return;
auto convert_actions_dag = getConvertingDAG(plan.getCurrentDataStream().header, header);
auto converting = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), convert_actions_dag);
plan.addStep(std::move(converting));
}
std::unique_ptr<QueryPlan> createLocalPlan(
const ASTPtr & query_ast,
const Block & header,
ContextPtr context,
QueryProcessingStage::Enum processed_stage,
UInt32 shard_num,
UInt32 shard_count)
{
checkStackSize();
auto query_plan = std::make_unique<QueryPlan>();
InterpreterSelectQuery interpreter(
query_ast, context, SelectQueryOptions(processed_stage).setShardInfo(shard_num, shard_count));
interpreter.buildQueryPlan(*query_plan);
addConvertingActions(*query_plan, header);
return query_plan;
}
}
void SelectStreamFactory::createForShard(
const Cluster::ShardInfo & shard_info,
const ASTPtr & query_ast,
@ -111,7 +63,7 @@ void SelectStreamFactory::createForShard(
auto emplace_local_stream = [&]()
{
local_plans.emplace_back(createLocalPlan(query_ast, header, context, processed_stage, shard_info.shard_num, shard_count));
local_plans.emplace_back(createLocalPlan(query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, /*coordinator=*/nullptr));
};
auto emplace_remote_stream = [&](bool lazy = false, UInt32 local_delay = 0)

14
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -1672,6 +1672,20 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(
}
}
void InterpreterSelectQuery::setMergeTreeReadTaskCallbackAndClientInfo(MergeTreeReadTaskCallback && callback)
{
context->getClientInfo().collaborate_with_initiator = true;
context->setMergeTreeReadTaskCallback(std::move(callback));
}
void InterpreterSelectQuery::setProperClientInfo()
{
context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY;
assert(options.shard_count.has_value() && options.shard_num.has_value());
context->getClientInfo().count_participating_replicas = *options.shard_count;
context->getClientInfo().number_of_current_replica = *options.shard_num;
}
bool InterpreterSelectQuery::shouldMoveToPrewhere()
{
const Settings & settings = context->getSettingsRef();

10
src/Interpreters/InterpreterSelectQuery.h
View File

@ -108,6 +108,16 @@ public:
bool supportsTransactions() const override { return true; }
/// This is tiny crutch to support reading from localhost replica during distributed query
/// Replica need to talk to the initiator through a connection to ask for a next task
/// but there will be no connection if we create Interpreter explicitly.
/// The other problem is that context is copied inside Interpreter's constructor
/// And with this method we can change the internals of cloned one
void setMergeTreeReadTaskCallbackAndClientInfo(MergeTreeReadTaskCallback && callback);
/// It will set shard_num and shard_count to the client_info
void setProperClientInfo();
private:
InterpreterSelectQuery(
const ASTPtr & query_ptr_,

5
src/Interpreters/ProcessList.h
View File

@ -164,6 +164,11 @@ public:
return &thread_group->memory_tracker;
}
IAST::QueryKind getQueryKind() const
{
return query_kind;
}
bool updateProgressIn(const Progress & value)
{
CurrentThread::updateProgressIn(value);

4
src/Interpreters/SessionLog.cpp
View File

@ -105,7 +105,9 @@ NamesAndTypesList SessionLogElement::getNamesAndTypes()
{"HTTP", static_cast<Int8>(Interface::HTTP)},
{"gRPC", static_cast<Int8>(Interface::GRPC)},
{"MySQL", static_cast<Int8>(Interface::MYSQL)},
{"PostgreSQL", static_cast<Int8>(Interface::POSTGRESQL)}
{"PostgreSQL", static_cast<Int8>(Interface::POSTGRESQL)},
{"LOCAL", static_cast<Int8>(Interface::LOCAL)},
{"TCP_INTERSERVER", static_cast<Int8>(Interface::TCP_INTERSERVER)}
});
auto lc_string_datatype = std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>());

306
src/Parsers/ExpressionElementParsers.cpp
View File

@ -298,28 +298,69 @@ namespace
{
bool parseCastAs(IParser::Pos & pos, ASTPtr & node, Expected & expected)
{
/// expr AS type
/** Possible variants for cast operator cast(expr [[AS] alias_1] AS Type), cast(expr [[AS] alias_1], type_expr [[as] alias_2]).
* First try to match with cast(expr [[AS] alias_1] AS Type)
* Then try to match with cast(expr [[AS] alias_1], type_expr [[as] alias_2]).
*/
ASTPtr expr_node;
ASTPtr type_node;
ASTPtr identifier_node;
if (ParserExpression().parse(pos, expr_node, expected))
{
if (ParserKeyword("AS").ignore(pos, expected))
ParserKeyword as_keyword_parser("AS");
bool parse_as = as_keyword_parser.ignore(pos, expected);
/// CAST (a b AS UInt32) OR CAST (a b, expr)
if (!parse_as && ParserIdentifier().parse(pos, identifier_node, expected))
{
if (ParserDataType().parse(pos, type_node, expected))
expr_node->setAlias(getIdentifierName(identifier_node));
parse_as = as_keyword_parser.ignore(pos, expected);
}
if (parse_as)
{
/// CAST (a AS Type) OR CAST (a AS b AS Type) OR CAST (a AS b, expr)
auto begin = pos;
auto expected_copy = expected;
bool next_identifier = ParserIdentifier().ignore(begin, expected_copy);
bool next_identifier_with_comma = next_identifier && ParserToken(TokenType::Comma).ignore(begin, expected_copy);
bool next_identifier_with_as
= next_identifier && !next_identifier_with_comma && as_keyword_parser.ignore(begin, expected_copy);
if (next_identifier_with_as)
{
if (ParserIdentifier().parse(pos, identifier_node, expected) && as_keyword_parser.ignore(pos, expected))
expr_node->setAlias(getIdentifierName(identifier_node));
else
return false;
}
if (!next_identifier_with_comma && ParserDataType().parse(pos, type_node, expected))
{
node = createFunctionCast(expr_node, type_node);
return true;
}
}
else if (ParserToken(TokenType::Comma).ignore(pos, expected))
/// CAST(a AS b, expr)
if (parse_as)
{
if (ParserExpression().parse(pos, type_node, expected))
{
node = makeASTFunction("CAST", expr_node, type_node);
return true;
}
if (ParserIdentifier().parse(pos, identifier_node, expected))
expr_node->setAlias(getIdentifierName(identifier_node));
else
return false;
}
if (ParserToken(TokenType::Comma).ignore(pos, expected)
&& ParserExpressionWithOptionalAlias(true /*allow_alias_without_as_keyword*/).parse(pos, type_node, expected))
{
node = makeASTFunction("CAST", expr_node, type_node);
return true;
}
}
@ -331,6 +372,9 @@ namespace
/// Either SUBSTRING(expr FROM start) or SUBSTRING(expr FROM start FOR length) or SUBSTRING(expr, start, length)
/// The latter will be parsed normally as a function later.
ParserKeyword as_keyword_parser("AS");
ParserIdentifier identifier_parser;
ASTPtr expr_node;
ASTPtr start_node;
ASTPtr length_node;
@ -338,35 +382,65 @@ namespace
if (!ParserExpression().parse(pos, expr_node, expected))
return false;
if (pos->type != TokenType::Comma)
auto from_keyword_parser = ParserKeyword("FROM");
bool from_exists = from_keyword_parser.check(pos, expected);
if (!from_exists && pos->type != TokenType::Comma)
{
if (!ParserKeyword("FROM").ignore(pos, expected))
ASTPtr identifier_node;
bool parsed_as = as_keyword_parser.ignore(pos, expected);
bool parsed_identifer = identifier_parser.parse(pos, identifier_node, expected);
if (parsed_as && !parsed_identifer)
return false;
if (parsed_identifer)
expr_node->setAlias(getIdentifierName(identifier_node));
from_exists = from_keyword_parser.check(pos, expected);
}
else
if (pos->type == TokenType::Comma)
{
if (from_exists)
return false;
++pos;
}
if (!ParserExpression().parse(pos, start_node, expected))
return false;
if (pos->type != TokenType::ClosingRoundBracket)
auto for_keyword_parser = ParserKeyword("FOR");
bool for_exists = for_keyword_parser.check(pos, expected);
if (!for_exists && pos->type != TokenType::Comma)
{
if (pos->type != TokenType::Comma)
{
if (!ParserKeyword("FOR").ignore(pos, expected))
return false;
}
else
{
++pos;
}
if (!ParserExpression().parse(pos, length_node, expected))
ASTPtr identifier_node;
bool parsed_as = as_keyword_parser.ignore(pos, expected);
bool parsed_identifer = identifier_parser.parse(pos, identifier_node, expected);
if (parsed_as && !parsed_identifer)
return false;
if (parsed_identifer)
start_node->setAlias(getIdentifierName(identifier_node));
for_exists = for_keyword_parser.check(pos, expected);
}
bool need_parse_length_expression = for_exists;
if (pos->type == TokenType::Comma)
{
if (for_exists)
return false;
++pos;
need_parse_length_expression = true;
}
if (need_parse_length_expression
&& !ParserExpressionWithOptionalAlias(true /*allow_alias_without_as_keyword*/).parse(pos, length_node, expected))
return false;
/// Convert to canonical representation in functional form: SUBSTRING(expr, start, length)
if (length_node)
node = makeASTFunction("substring", expr_node, start_node, length_node);
@ -378,7 +452,7 @@ namespace
bool parseTrim(bool trim_left, bool trim_right, IParser::Pos & pos, ASTPtr & node, Expected & expected)
{
/// Handles all possible TRIM/LTRIM/RTRIM call variants
/// Handles all possible TRIM/LTRIM/RTRIM call variants ([[LEADING|TRAILING|BOTH] trim_character FROM] input_string)
std::string func_name;
bool char_override = false;
@ -414,7 +488,26 @@ namespace
{
if (!ParserExpression().parse(pos, to_remove, expected))
return false;
if (!ParserKeyword("FROM").ignore(pos, expected))
auto from_keyword_parser = ParserKeyword("FROM");
bool from_exists = from_keyword_parser.check(pos, expected);
if (!from_exists)
{
ASTPtr identifier_node;
bool parsed_as = ParserKeyword("AS").ignore(pos, expected);
bool parsed_identifer = ParserIdentifier().parse(pos, identifier_node, expected);
if (parsed_as && !parsed_identifer)
return false;
if (parsed_identifer)
to_remove->setAlias(getIdentifierName(identifier_node));
from_exists = from_keyword_parser.check(pos, expected);
}
if (!from_exists)
return false;
auto quote_meta_func_node = std::make_shared<ASTFunction>();
@ -429,7 +522,7 @@ namespace
}
}
if (!ParserExpression().parse(pos, expr_node, expected))
if (!ParserExpressionWithOptionalAlias(true /*allow_alias_without_as_keyword*/).parse(pos, expr_node, expected))
return false;
/// Convert to regexp replace function call
@ -440,33 +533,24 @@ namespace
auto pattern_list_args = std::make_shared<ASTExpressionList>();
if (trim_left && trim_right)
{
pattern_list_args->children = {
std::make_shared<ASTLiteral>("^["),
to_remove,
std::make_shared<ASTLiteral>("]+|["),
to_remove,
std::make_shared<ASTLiteral>("]+$")
};
pattern_list_args->children
= {std::make_shared<ASTLiteral>("^["),
to_remove,
std::make_shared<ASTLiteral>("]+|["),
to_remove,
std::make_shared<ASTLiteral>("]+$")};
func_name = "replaceRegexpAll";
}
else
{
if (trim_left)
{
pattern_list_args->children = {
std::make_shared<ASTLiteral>("^["),
to_remove,
std::make_shared<ASTLiteral>("]+")
};
pattern_list_args->children = {std::make_shared<ASTLiteral>("^["), to_remove, std::make_shared<ASTLiteral>("]+")};
}
else
{
/// trim_right == false not possible
pattern_list_args->children = {
std::make_shared<ASTLiteral>("["),
to_remove,
std::make_shared<ASTLiteral>("]+$")
};
pattern_list_args->children = {std::make_shared<ASTLiteral>("["), to_remove, std::make_shared<ASTLiteral>("]+$")};
}
func_name = "replaceRegexpOne";
}
@ -506,6 +590,9 @@ namespace
bool parseExtract(IParser::Pos & pos, ASTPtr & node, Expected & expected)
{
/// First try to match with date extract operator EXTRACT(part FROM date)
/// Then with function extract(haystack, pattern)
IParser::Pos begin = pos;
IntervalKind interval_kind;
@ -514,7 +601,7 @@ namespace
ASTPtr expr;
ParserKeyword s_from("FROM");
ParserExpression elem_parser;
ParserExpressionWithOptionalAlias elem_parser(true /*allow_alias_without_as_keyword*/);
if (s_from.ignore(pos, expected) && elem_parser.parse(pos, expr, expected))
{
@ -526,7 +613,7 @@ namespace
pos = begin;
ASTPtr expr_list;
if (!ParserExpressionList(false, false).parse(pos, expr_list, expected))
if (!ParserExpressionList(true /*allow_alias_without_as_keyword*/).parse(pos, expr_list, expected))
return false;
auto res = std::make_shared<ASTFunction>();
@ -539,28 +626,57 @@ namespace
bool parsePosition(IParser::Pos & pos, ASTPtr & node, Expected & expected)
{
ASTPtr expr_list_node;
if (!ParserExpressionList(false, false).parse(pos, expr_list_node, expected))
return false;
/// First try to match with position(needle IN haystack)
/// Then with position(haystack, needle[, start_pos])
ASTExpressionList * expr_list = typeid_cast<ASTExpressionList *>(expr_list_node.get());
if (expr_list && expr_list->children.size() == 1)
ParserExpressionWithOptionalAlias expr_parser(true /*allow_alias_without_as_keyword*/);
ASTPtr first_arg_expr_node;
if (!expr_parser.parse(pos, first_arg_expr_node, expected))
{
ASTFunction * func_in = typeid_cast<ASTFunction *>(expr_list->children[0].get());
if (func_in && func_in->name == "in")
return false;
}
ASTFunction * func_in = typeid_cast<ASTFunction *>(first_arg_expr_node.get());
if (func_in && func_in->name == "in")
{
ASTExpressionList * in_args = typeid_cast<ASTExpressionList *>(func_in->arguments.get());
if (in_args && in_args->children.size() == 2)
{
ASTExpressionList * in_args = typeid_cast<ASTExpressionList *>(func_in->arguments.get());
if (in_args && in_args->children.size() == 2)
{
node = makeASTFunction("position", in_args->children[1], in_args->children[0]);
return true;
}
node = makeASTFunction("position", in_args->children[1], in_args->children[0]);
return true;
}
}
if (pos->type != TokenType::Comma)
return false;
++pos;
ASTPtr second_arg_expr_node;
if (!expr_parser.parse(pos, second_arg_expr_node, expected))
{
return false;
}
ASTPtr start_pos_expr_node;
if (pos->type == TokenType::Comma)
{
++pos;
if (!expr_parser.parse(pos, start_pos_expr_node, expected))
return false;
}
auto arguments = std::make_shared<ASTExpressionList>();
arguments->children.push_back(std::move(first_arg_expr_node));
arguments->children.push_back(std::move(second_arg_expr_node));
if (start_pos_expr_node)
arguments->children.push_back(std::move(start_pos_expr_node));
auto res = std::make_shared<ASTFunction>();
res->name = "position";
res->arguments = expr_list_node;
res->arguments = std::move(arguments);
res->children.push_back(res->arguments);
node = std::move(res);
return true;
@ -568,6 +684,9 @@ namespace
bool parseDateAdd(const char * function_name, IParser::Pos & pos, ASTPtr & node, Expected & expected)
{
/// First to match with function(unit, offset, timestamp)
/// Then with function(offset, timestamp)
ASTPtr timestamp_node;
ASTPtr offset_node;
@ -575,19 +694,18 @@ namespace
ASTPtr interval_func_node;
if (parseIntervalKind(pos, expected, interval_kind))
{
/// function(unit, offset, timestamp)
if (pos->type != TokenType::Comma)
return false;
++pos;
if (!ParserExpression().parse(pos, offset_node, expected))
if (!ParserExpressionWithOptionalAlias(true /*allow_alias_without_as_keyword*/).parse(pos, offset_node, expected))
return false;
if (pos->type != TokenType::Comma)
return false;
++pos;
if (!ParserExpression().parse(pos, timestamp_node, expected))
if (!ParserExpressionWithOptionalAlias(true /*allow_alias_without_as_keyword*/).parse(pos, timestamp_node, expected))
return false;
auto interval_expr_list_args = std::make_shared<ASTExpressionList>();
interval_expr_list_args->children = {offset_node};
@ -600,7 +718,7 @@ namespace
else
{
ASTPtr expr_list;
if (!ParserExpressionList(false, false).parse(pos, expr_list, expected))
if (!ParserExpressionList(true /*allow_alias_without_as_keyword*/).parse(pos, expr_list, expected))
return false;
auto res = std::make_shared<ASTFunction>();
@ -617,39 +735,59 @@ namespace
bool parseDateDiff(IParser::Pos & pos, ASTPtr & node, Expected & expected)
{
/// First to match with dateDiff(unit, startdate, enddate, [timezone])
/// Then with dateDiff('unit', startdate, enddate, [timezone])
ASTPtr left_node;
ASTPtr right_node;
IntervalKind interval_kind;
if (!parseIntervalKind(pos, expected, interval_kind))
if (parseIntervalKind(pos, expected, interval_kind))
{
ASTPtr expr_list;
if (!ParserExpressionList(false, false).parse(pos, expr_list, expected))
if (pos->type != TokenType::Comma)
return false;
++pos;
if (!ParserExpressionWithOptionalAlias(true /*allow_alias_without_as_keyword*/).parse(pos, left_node, expected))
return false;
auto res = std::make_shared<ASTFunction>();
res->name = "dateDiff";
res->arguments = expr_list;
res->children.push_back(res->arguments);
node = std::move(res);
if (pos->type != TokenType::Comma)
return false;
++pos;
if (!ParserExpressionWithOptionalAlias(true /*allow_alias_without_as_keyword*/).parse(pos, right_node, expected))
return false;
ASTPtr timezone_node;
if (pos->type == TokenType::Comma)
{
/// Optional timezone
++pos;
if (!ParserExpressionWithOptionalAlias(true /*allow_alias_without_as_keyword*/).parse(pos, timezone_node, expected))
return false;
}
auto interval_literal = std::make_shared<ASTLiteral>(interval_kind.toDateDiffUnit());
if (timezone_node)
node = makeASTFunction("dateDiff", std::move(interval_literal), std::move(left_node), std::move(right_node), std::move(timezone_node));
else
node = makeASTFunction("dateDiff", std::move(interval_literal), std::move(left_node), std::move(right_node));
return true;
}
if (pos->type != TokenType::Comma)
return false;
++pos;
if (!ParserExpression().parse(pos, left_node, expected))
ASTPtr expr_list;
if (!ParserExpressionList(true /*allow_alias_without_as_keyword*/).parse(pos, expr_list, expected))
return false;
if (pos->type != TokenType::Comma)
return false;
++pos;
auto res = std::make_shared<ASTFunction>();
res->name = "dateDiff";
res->arguments = expr_list;
res->children.push_back(res->arguments);
node = std::move(res);
if (!ParserExpression().parse(pos, right_node, expected))
return false;
node = makeASTFunction("dateDiff", std::make_shared<ASTLiteral>(interval_kind.toDateDiffUnit()), left_node, right_node);
return true;
}

16
src/Parsers/ParserDataType.cpp
View File

@ -92,11 +92,25 @@ bool ParserDataType::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
}
else if (type_name_upper.find("INT") != std::string::npos)
{
/// Support SIGNED and UNSIGNED integer type modifiers for compatibility with MySQL.
/// Support SIGNED and UNSIGNED integer type modifiers for compatibility with MySQL
if (ParserKeyword("SIGNED").ignore(pos))
type_name_suffix = "SIGNED";
else if (ParserKeyword("UNSIGNED").ignore(pos))
type_name_suffix = "UNSIGNED";
else if (pos->type == TokenType::OpeningRoundBracket)
{
++pos;
if (pos->type == TokenType::Number)
++pos;
if (pos->type != TokenType::ClosingRoundBracket)
return false;
++pos;
if (ParserKeyword("SIGNED").ignore(pos))
type_name_suffix = "SIGNED";
else if (ParserKeyword("UNSIGNED").ignore(pos))
type_name_suffix = "UNSIGNED";
}
}
if (!type_name_suffix.empty())

1
src/Processors/Formats/IInputFormat.h
View File

@ -51,6 +51,7 @@ public:
virtual void resetParser();
virtual void setReadBuffer(ReadBuffer & in_);
const ReadBuffer & getReadBuffer() const { return *in; }
virtual const BlockMissingValues & getMissingValues() const
{

9
src/Processors/Formats/IRowInputFormat.cpp
View File

@ -1,6 +1,7 @@
#include <Processors/Formats/IRowInputFormat.h>
#include <DataTypes/ObjectUtils.h>
#include <IO/WriteHelpers.h> // toString
#include <IO/WithFileName.h>
#include <base/logger_useful.h>
@ -165,6 +166,7 @@ Chunk IRowInputFormat::generate()
/// Error while trying to obtain verbose diagnostic. Ok to ignore.
}
e.setFileName(getFileNameFromReadBuffer(getReadBuffer()));
e.setLineNumber(total_rows);
e.addMessage(verbose_diagnostic);
throw;
@ -188,7 +190,12 @@ Chunk IRowInputFormat::generate()
/// Error while trying to obtain verbose diagnostic. Ok to ignore.
}
e.addMessage("(at row " + toString(total_rows) + ")\n" + verbose_diagnostic);
auto file_name = getFileNameFromReadBuffer(getReadBuffer());
if (!file_name.empty())
e.addMessage(fmt::format("(in file/uri {})", file_name));
e.addMessage(fmt::format("(at row {})\n", total_rows));
e.addMessage(verbose_diagnostic);
throw;
}

32
src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
View File

@ -360,6 +360,24 @@ void registerInputFormatJSONEachRow(FormatFactory & factory)
return std::make_shared<JSONEachRowRowInputFormat>(buf, sample, std::move(params), settings, false);
});
factory.registerInputFormat("JSONLines", [](
ReadBuffer & buf,
const Block & sample,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
return std::make_shared<JSONEachRowRowInputFormat>(buf, sample, std::move(params), settings, false);
});
factory.registerInputFormat("NDJSON", [](
ReadBuffer & buf,
const Block & sample,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
return std::make_shared<JSONEachRowRowInputFormat>(buf, sample, std::move(params), settings, false);
});
factory.registerFileExtension("ndjson", "JSONEachRow");
factory.registerFileExtension("jsonl", "JSONEachRow");
@ -377,12 +395,16 @@ void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory)
{
factory.registerFileSegmentationEngine("JSONEachRow", &fileSegmentationEngineJSONEachRow);
factory.registerFileSegmentationEngine("JSONStringsEachRow", &fileSegmentationEngineJSONEachRow);
factory.registerFileSegmentationEngine("JSONLines", &fileSegmentationEngineJSONEachRow);
factory.registerFileSegmentationEngine("NDJSON", &fileSegmentationEngineJSONEachRow);
}
void registerNonTrivialPrefixAndSuffixCheckerJSONEachRow(FormatFactory & factory)
{
factory.registerNonTrivialPrefixAndSuffixChecker("JSONEachRow", nonTrivialPrefixAndSuffixCheckerJSONEachRowImpl);
factory.registerNonTrivialPrefixAndSuffixChecker("JSONStringsEachRow", nonTrivialPrefixAndSuffixCheckerJSONEachRowImpl);
factory.registerNonTrivialPrefixAndSuffixChecker("JSONLines", nonTrivialPrefixAndSuffixCheckerJSONEachRowImpl);
factory.registerNonTrivialPrefixAndSuffixChecker("NDJSON", nonTrivialPrefixAndSuffixCheckerJSONEachRowImpl);
}
void registerJSONEachRowSchemaReader(FormatFactory & factory)
@ -396,6 +418,16 @@ void registerJSONEachRowSchemaReader(FormatFactory & factory)
{
return std::make_unique<JSONEachRowSchemaReader>(buf, true, settings);
});
factory.registerSchemaReader("JSONLines", [](ReadBuffer & buf, const FormatSettings & settings)
{
return std::make_unique<JSONEachRowSchemaReader>(buf, false, settings);
});
factory.registerSchemaReader("NDJSON", [](ReadBuffer & buf, const FormatSettings & settings)
{
return std::make_unique<JSONEachRowSchemaReader>(buf, false, settings);
});
}
}

26
src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp
View File

@ -140,6 +140,32 @@ void registerOutputFormatJSONEachRow(FormatFactory & factory)
});
factory.markOutputFormatSupportsParallelFormatting("JSONEachRow");
factory.registerOutputFormat("JSONLines", [](
WriteBuffer & buf,
const Block & sample,
const RowOutputFormatParams & params,
const FormatSettings & _format_settings)
{
FormatSettings settings = _format_settings;
settings.json.serialize_as_strings = false;
return std::make_shared<JSONEachRowRowOutputFormat>(buf, sample, params,
settings);
});
factory.markOutputFormatSupportsParallelFormatting("JSONLines");
factory.registerOutputFormat("NDJSON", [](
WriteBuffer & buf,
const Block & sample,
const RowOutputFormatParams & params,
const FormatSettings & _format_settings)
{
FormatSettings settings = _format_settings;
settings.json.serialize_as_strings = false;
return std::make_shared<JSONEachRowRowOutputFormat>(buf, sample, params,
settings);
});
factory.markOutputFormatSupportsParallelFormatting("NDJSON");
factory.registerOutputFormat("JSONStringsEachRow", [](
WriteBuffer & buf,
const Block & sample,

9
src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp
View File

@ -1,5 +1,6 @@
#include <Processors/Formats/Impl/ParallelParsingInputFormat.h>
#include <IO/ReadHelpers.h>
#include <IO/WithFileName.h>
#include <Common/CurrentThread.h>
#include <Common/setThreadName.h>
@ -125,11 +126,19 @@ void ParallelParsingInputFormat::onBackgroundException(size_t offset)
{
background_exception = std::current_exception();
if (ParsingException * e = exception_cast<ParsingException *>(background_exception))
{
if (e->getLineNumber() != -1)
e->setLineNumber(e->getLineNumber() + offset);
auto file_name = getFileNameFromReadBuffer(getReadBuffer());
if (!file_name.empty())
e->setFileName(file_name);
}
}
if (is_server)
tryLogCurrentException(__PRETTY_FUNCTION__);
parsing_finished = true;
first_parser_finished.set();
reader_condvar.notify_all();

66
src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp Normal file
View File

@ -0,0 +1,66 @@
#include <Processors/QueryPlan/DistributedCreateLocalPlan.h>
#include <Common/checkStackSize.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Interpreters/ActionsDAG.h>
#include <Interpreters/InterpreterSelectQuery.h>
namespace DB
{
namespace
{
void addConvertingActions(QueryPlan & plan, const Block & header)
{
if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header))
return;
auto get_converting_dag = [](const Block & block_, const Block & header_)
{
/// Convert header structure to expected.
/// Also we ignore constants from result and replace it with constants from header.
/// It is needed for functions like `now64()` or `randConstant()` because their values may be different.
return ActionsDAG::makeConvertingActions(
block_.getColumnsWithTypeAndName(),
header_.getColumnsWithTypeAndName(),
ActionsDAG::MatchColumnsMode::Name,
true);
};
auto convert_actions_dag = get_converting_dag(plan.getCurrentDataStream().header, header);
auto converting = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), convert_actions_dag);
plan.addStep(std::move(converting));
}
}
std::unique_ptr<QueryPlan> createLocalPlan(
const ASTPtr & query_ast,
const Block & header,
ContextPtr context,
QueryProcessingStage::Enum processed_stage,
UInt32 shard_num,
UInt32 shard_count,
std::shared_ptr<ParallelReplicasReadingCoordinator> coordinator)
{
checkStackSize();
auto query_plan = std::make_unique<QueryPlan>();
auto interpreter = InterpreterSelectQuery(
query_ast, context, SelectQueryOptions(processed_stage).setShardInfo(shard_num, shard_count));
interpreter.setProperClientInfo();
if (coordinator)
{
interpreter.setMergeTreeReadTaskCallbackAndClientInfo([coordinator](PartitionReadRequest request) -> std::optional<PartitionReadResponse>
{
return coordinator->handleRequest(request);
});
}
interpreter.buildQueryPlan(*query_plan);
addConvertingActions(*query_plan, header);
return query_plan;
}
}

20
src/Processors/QueryPlan/DistributedCreateLocalPlan.h Normal file
View File

@ -0,0 +1,20 @@
#pragma once
#include <Core/QueryProcessingStage.h>
#include <Parsers/IAST_fwd.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Storages/MergeTree/ParallelReplicasReadingCoordinator.h>
namespace DB
{
std::unique_ptr<QueryPlan> createLocalPlan(
const ASTPtr & query_ast,
const Block & header,
ContextPtr context,
QueryProcessingStage::Enum processed_stage,
UInt32 shard_num,
UInt32 shard_count,
std::shared_ptr<ParallelReplicasReadingCoordinator> coordinator);
}

65
src/Processors/QueryPlan/ReadFromRemote.cpp
View File

@ -1,6 +1,7 @@
#include <Processors/QueryPlan/ReadFromRemote.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/DistributedCreateLocalPlan.h>
#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
#include <QueryPipeline/RemoteQueryExecutor.h>
#include <Parsers/ASTSelectQuery.h>
@ -23,34 +24,24 @@ namespace ErrorCodes
extern const int ALL_CONNECTION_TRIES_FAILED;
}
static ActionsDAGPtr getConvertingDAG(const Block & block, const Block & header)
{
/// Convert header structure to expected.
/// Also we ignore constants from result and replace it with constants from header.
/// It is needed for functions like `now64()` or `randConstant()` because their values may be different.
return ActionsDAG::makeConvertingActions(
block.getColumnsWithTypeAndName(),
header.getColumnsWithTypeAndName(),
ActionsDAG::MatchColumnsMode::Name,
true);
}
void addConvertingActions(QueryPlan & plan, const Block & header)
{
if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header))
return;
auto convert_actions_dag = getConvertingDAG(plan.getCurrentDataStream().header, header);
auto converting = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), convert_actions_dag);
plan.addStep(std::move(converting));
}
static void addConvertingActions(Pipe & pipe, const Block & header)
{
if (blocksHaveEqualStructure(pipe.getHeader(), header))
return;
auto convert_actions = std::make_shared<ExpressionActions>(getConvertingDAG(pipe.getHeader(), header));
auto get_converting_dag = [](const Block & block_, const Block & header_)
{
/// Convert header structure to expected.
/// Also we ignore constants from result and replace it with constants from header.
/// It is needed for functions like `now64()` or `randConstant()` because their values may be different.
return ActionsDAG::makeConvertingActions(
block_.getColumnsWithTypeAndName(),
header_.getColumnsWithTypeAndName(),
ActionsDAG::MatchColumnsMode::Name,
true);
};
auto convert_actions = std::make_shared<ExpressionActions>(get_converting_dag(pipe.getHeader(), header));
pipe.addSimpleTransform([&](const Block & cur_header, Pipe::StreamType) -> ProcessorPtr
{
return std::make_shared<ExpressionTransform>(cur_header, convert_actions);
@ -70,28 +61,6 @@ static String formattedAST(const ASTPtr & ast)
return buf.str();
}
static std::unique_ptr<QueryPlan> createLocalPlan(
const ASTPtr & query_ast,
const Block & header,
ContextPtr context,
QueryProcessingStage::Enum processed_stage,
UInt32 shard_num,
UInt32 shard_count)
{
checkStackSize();
auto query_plan = std::make_unique<QueryPlan>();
InterpreterSelectQuery interpreter(
query_ast, context, SelectQueryOptions(processed_stage).setShardInfo(shard_num, shard_count));
interpreter.buildQueryPlan(*query_plan);
addConvertingActions(*query_plan, header);
return query_plan;
}
ReadFromRemote::ReadFromRemote(
ClusterProxy::IStreamFactory::Shards shards_,
Block header_,
@ -173,11 +142,9 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::IStreamFacto
max_remote_delay = std::max(try_result.staleness, max_remote_delay);
}
/// We disable this branch in case of parallel reading from replicas, because createLocalPlan will call
/// InterpreterSelectQuery directly and it will be too ugly to pass ParallelReplicasCoordinator or some callback there.
if (!context->getClientInfo().collaborate_with_initiator && (try_results.empty() || local_delay < max_remote_delay))
if (try_results.empty() || local_delay < max_remote_delay)
{
auto plan = createLocalPlan(query, header, context, stage, shard_num, shard_count);
auto plan = createLocalPlan(query, header, context, stage, shard_num, shard_count, coordinator);
return QueryPipelineBuilder::getPipe(std::move(*plan->buildQueryPipeline(
QueryPlanOptimizationSettings::fromContext(context),

14
src/Server/TCPHandler.cpp
View File

@ -202,7 +202,7 @@ void TCPHandler::runImpl()
/** An exception during the execution of request (it must be sent over the network to the client).
* The client will be able to accept it, if it did not happen while sending another packet and the client has not disconnected yet.
*/
std::optional<DB::Exception> exception;
std::unique_ptr<DB::Exception> exception;
bool network_error = false;
try
@ -396,7 +396,7 @@ void TCPHandler::runImpl()
catch (const Exception & e)
{
state.io.onException();
exception.emplace(e);
exception.reset(e.clone());
if (e.code() == ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT)
throw;
@ -420,12 +420,12 @@ void TCPHandler::runImpl()
* We will try to send exception to the client in any case - see below.
*/
state.io.onException();
exception.emplace(Exception::CreateFromPocoTag{}, e);
exception = std::make_unique<DB::Exception>(Exception::CreateFromPocoTag{}, e);
}
catch (const Poco::Exception & e)
{
state.io.onException();
exception.emplace(Exception::CreateFromPocoTag{}, e);
exception = std::make_unique<DB::Exception>(Exception::CreateFromPocoTag{}, e);
}
// Server should die on std logic errors in debug, like with assert()
// or ErrorCodes::LOGICAL_ERROR. This helps catch these errors in
@ -434,7 +434,7 @@ void TCPHandler::runImpl()
catch (const std::logic_error & e)
{
state.io.onException();
exception.emplace(Exception::CreateFromSTDTag{}, e);
exception = std::make_unique<DB::Exception>(Exception::CreateFromSTDTag{}, e);
sendException(*exception, send_exception_with_stack_trace);
std::abort();
}
@ -442,12 +442,12 @@ void TCPHandler::runImpl()
catch (const std::exception & e)
{
state.io.onException();
exception.emplace(Exception::CreateFromSTDTag{}, e);
exception = std::make_unique<DB::Exception>(Exception::CreateFromSTDTag{}, e);
}
catch (...)
{
state.io.onException();
exception.emplace("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
exception = std::make_unique<DB::Exception>("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
}
try

5
src/Storages/HDFS/ReadBufferFromHDFS.cpp
View File

@ -182,6 +182,11 @@ size_t ReadBufferFromHDFS::getFileOffsetOfBufferEnd() const
return impl->getPosition();
}
String ReadBufferFromHDFS::getFileName() const
{
return impl->hdfs_file_path;
}
}
#endif

5
src/Storages/HDFS/ReadBufferFromHDFS.h
View File

@ -11,6 +11,7 @@
#include <base/types.h>
#include <Interpreters/Context.h>
#include <IO/SeekableReadBuffer.h>
#include <IO/WithFileName.h>
namespace DB
@ -19,7 +20,7 @@ namespace DB
/** Accepts HDFS path to file and opens it.
* Closes file by himself (thus "owns" a file descriptor).
*/
class ReadBufferFromHDFS : public SeekableReadBufferWithSize
class ReadBufferFromHDFS : public SeekableReadBufferWithSize, public WithFileName
{
struct ReadBufferFromHDFSImpl;
@ -41,6 +42,8 @@ public:
size_t getFileOffsetOfBufferEnd() const override;
String getFileName() const override;
private:
std::unique_ptr<ReadBufferFromHDFSImpl> impl;
};

1
src/Storages/StorageFile.cpp
View File

@ -544,7 +544,6 @@ public:
Chunk chunk;
if (reader->pull(chunk))
{
//Columns columns = res.getColumns();
UInt64 num_rows = chunk.getNumRows();
/// Enrich with virtual columns.

24
src/Storages/System/StorageSystemTables.cpp
View File

@ -244,10 +244,30 @@ protected:
if (columns_mask[src_index++])
res_columns[res_index++]->insert(table.second->getName());
/// Fill the rest columns with defaults
const auto & settings = context->getSettingsRef();
while (src_index < columns_mask.size())
if (columns_mask[src_index++])
{
// total_rows
if (src_index == 18 && columns_mask[src_index])
{
if (auto total_rows = table.second->totalRows(settings))
res_columns[res_index++]->insert(*total_rows);
else
res_columns[res_index++]->insertDefault();
}
// total_bytes
else if (src_index == 19 && columns_mask[src_index])
{
if (auto total_bytes = table.second->totalBytes(settings))
res_columns[res_index++]->insert(*total_bytes);
else
res_columns[res_index++]->insertDefault();
}
/// Fill the rest columns with defaults
else if (columns_mask[src_index])
res_columns[res_index++]->insertDefault();
src_index++;
}
}
}

26
src/Storages/System/StorageSystemZooKeeper.cpp
View File

@ -83,11 +83,11 @@ static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context,
if (function->name == "and")
{
const auto & children = function->arguments->children;
return std::any_of(
std::begin(children),
std::end(children),
[&](const auto & child) { return extractPathImpl(*child, res, context, allow_unrestricted); });
for (const auto & child : function->arguments->children)
if (extractPathImpl(*child, res, context, allow_unrestricted))
return true;
return false;
}
const auto & args = function->arguments->as<ASTExpressionList &>();
@ -136,11 +136,8 @@ static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context,
}
else if (Tuple tuple; literal->value.tryGet(tuple))
{
std::transform(
std::begin(tuple),
std::end(tuple),
std::back_inserter(res),
[](auto element) { return std::make_pair(element.template safeGet<String>(), ZkPathType::Exact); });
for (auto element : tuple)
res.emplace_back(element.safeGet<String>(), ZkPathType::Exact);
}
else
return false;
@ -289,12 +286,9 @@ void StorageSystemZooKeeper::fillData(MutableColumns & res_columns, ContextPtr c
}
std::vector<std::future<Coordination::GetResponse>> futures;
futures.reserve(std::size(nodes));
std::transform(
std::begin(nodes),
std::end(nodes),
std::back_inserter(futures),
[&zookeeper, &path_part](const String & node) { return zookeeper->asyncTryGet(path_part + '/' + node); });
futures.reserve(nodes.size());
for (const String & node : nodes)
futures.push_back(zookeeper->asyncTryGet(path_part + '/' + node));
for (size_t i = 0, size = nodes.size(); i < size; ++i)
{

10
tests/ci/build_check.py
View File

@ -14,6 +14,7 @@ from s3_helper import S3Helper
from pr_info import PRInfo
from version_helper import (
ClickHouseVersion,
Git,
get_version_from_repo,
update_version_local,
)
@ -209,7 +210,7 @@ def main():
s3_helper = S3Helper("https://s3.amazonaws.com")
version = get_version_from_repo()
version = get_version_from_repo(git=Git(True))
release_or_pr = get_release_or_pr(pr_info, build_config, version)
s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))
@ -284,9 +285,12 @@ def main():
if build_config["package_type"] == "performance" and pr_info.number != 0:
# because perf tests store some information about git commits
subprocess.check_call(
f"cd {REPO_COPY} && git fetch origin master:master", shell=True
cmd = (
f"cd {REPO_COPY} && git fetch --depth=60 --no-recurse-submodules "
"--no-tags origin master:master"
)
logging.info("Fetch master branch with a command: %s", cmd)
subprocess.check_call(cmd, shell=True)
packager_cmd = get_packager_cmd(
build_config,

6
tests/ci/fast_test_check.py
View File

@ -9,7 +9,7 @@ import sys
from github import Github
from env_helper import CACHES_PATH, TEMP_PATH
from pr_info import PRInfo
from pr_info import FORCE_TESTS_LABEL, PRInfo
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from upload_result_helper import upload_results
@ -214,7 +214,7 @@ if __name__ == "__main__":
# Refuse other checks to run if fast test failed
if state != "success":
if "force-tests" in pr_info.labels and state != "error":
print("'force-tests' enabled, will report success")
if FORCE_TESTS_LABEL in pr_info.labels and state != "error":
print(f"'{FORCE_TESTS_LABEL}' enabled, will report success")
else:
sys.exit(1)

6
tests/ci/functional_test_check.py
View File

@ -12,7 +12,7 @@ from github import Github
from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from pr_info import FORCE_TESTS_LABEL, PRInfo
from build_download_helper import download_all_deb_packages
from download_previous_release import download_previous_release
from upload_result_helper import upload_results
@ -359,7 +359,7 @@ if __name__ == "__main__":
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
if state != "success":
if "force-tests" in pr_info.labels:
print("'force-tests' enabled, will report success")
if FORCE_TESTS_LABEL in pr_info.labels:
print(f"'{FORCE_TESTS_LABEL}' enabled, will report success")
else:
sys.exit(1)

71
tests/ci/git_helper.py
View File

@ -13,6 +13,9 @@ TAG_REGEXP = (
)
SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z"
CWD = p.dirname(p.realpath(__file__))
TWEAK = 1
# Py 3.8 removeprefix and removesuffix
def removeprefix(string: str, prefix: str):
@ -46,8 +49,8 @@ def release_branch(name: str):
class Runner:
"""lightweight check_output wrapper with stripping last NEW_LINE"""
def __init__(self, cwd: str = p.dirname(p.realpath(__file__))):
self.cwd = cwd
def __init__(self, cwd: str = CWD):
self._cwd = cwd
def run(self, cmd: str, cwd: Optional[str] = None) -> str:
if cwd is None:
@ -56,22 +59,47 @@ class Runner:
cmd, shell=True, cwd=cwd, encoding="utf-8"
).strip()
@property
def cwd(self) -> str:
return self._cwd
@cwd.setter
def cwd(self, value: str):
# Set _cwd only once, then set it to readonly
if self._cwd != CWD:
return
self._cwd = value
git_runner = Runner()
# Set cwd to abs path of git root
git_runner.cwd = p.relpath(
p.join(git_runner.cwd, git_runner.run("git rev-parse --show-cdup"))
)
def get_tags() -> List[str]:
if git_runner.run("git rev-parse --is-shallow-repository") == "true":
raise RuntimeError("attempt to run on a shallow repository")
return git_runner.run("git tag").split()
class Git:
"""A small wrapper around subprocess to invoke git commands"""
def __init__(self):
runner = Runner()
rel_root = runner.run("git rev-parse --show-cdup")
self.root = p.realpath(p.join(runner.cwd, rel_root))
self._tag_pattern = re.compile(TAG_REGEXP)
runner.cwd = self.root
self.run = runner.run
_tag_pattern = re.compile(TAG_REGEXP)
def __init__(self, ignore_no_tags: bool = False):
self.root = git_runner.cwd
self._ignore_no_tags = ignore_no_tags
self.run = git_runner.run
self.latest_tag = ""
self.new_tag = ""
self.new_branch = ""
self.branch = ""
self.sha = ""
self.sha_short = ""
self.description = ""
self.description = "shallow-checkout"
self.commits_since_tag = 0
self.update()
@ -82,6 +110,19 @@ class Git:
self.sha_short = self.sha[:11]
# The following command shows the most recent tag in a graph
# Format should match TAG_REGEXP
if (
self._ignore_no_tags
and self.run("git rev-parse --is-shallow-repository") == "true"
):
try:
self._update_tags()
except subprocess.CalledProcessError:
pass
return
self._update_tags()
def _update_tags(self):
self.latest_tag = self.run("git describe --tags --abbrev=0")
# Format should be: {latest_tag}-{commits_since_tag}-g{sha_short}
self.description = self.run("git describe --tags --long")
@ -89,10 +130,11 @@ class Git:
self.run(f"git rev-list {self.latest_tag}..HEAD --count")
)
def check_tag(self, value: str):
@staticmethod
def check_tag(value: str):
if value == "":
return
if not self._tag_pattern.match(value):
if not Git._tag_pattern.match(value):
raise ValueError(f"last tag {value} doesn't match the pattern")
@property
@ -118,10 +160,7 @@ class Git:
if not self.latest_tag.endswith("-testing"):
# When we are on the tag, we still need to have tweak=1 to not
# break cmake with versions like 12.13.14.0
return self.commits_since_tag or 1
return self.commits_since_tag or TWEAK
version = self.latest_tag.split("-", maxsplit=1)[0]
return int(version.split(".")[-1]) + self.commits_since_tag
def get_tags(self) -> List[str]:
return self.run("git tag").split()

15
tests/ci/git_test.py
View File

@ -4,7 +4,7 @@ from unittest.mock import patch
import os.path as p
import unittest
from git_helper import Git, Runner
from git_helper import Git, Runner, CWD
class TestRunner(unittest.TestCase):
@ -19,6 +19,18 @@ class TestRunner(unittest.TestCase):
output = runner.run("echo 1")
self.assertEqual(output, "1")
def test_one_time_writeable_cwd(self):
runner = Runner()
self.assertEqual(runner.cwd, CWD)
runner.cwd = "/bin"
self.assertEqual(runner.cwd, "/bin")
runner.cwd = "/"
self.assertEqual(runner.cwd, "/bin")
runner = Runner("/")
self.assertEqual(runner.cwd, "/")
runner.cwd = "/bin"
self.assertEqual(runner.cwd, "/")
class TestGit(unittest.TestCase):
def setUp(self):
@ -31,6 +43,7 @@ class TestGit(unittest.TestCase):
self.addCleanup(update_patcher.stop)
self.git = Git()
update_mock.assert_called_once()
self.git.run("test")
self.run_mock.assert_called_once()
self.git.new_branch = "NEW_BRANCH_NAME"
self.git.new_tag = "v21.12.333.22222-stable"

8
tests/ci/pr_info.py
View File

@ -13,6 +13,8 @@ from env_helper import (
GITHUB_EVENT_PATH,
)
FORCE_TESTS_LABEL = "force tests"
DIFF_IN_DOCUMENTATION_EXT = [
".html",
".md",
@ -256,7 +258,7 @@ class PRInfo:
def can_skip_builds_and_use_version_from_master(self):
# TODO: See a broken loop
if "force tests" in self.labels:
if FORCE_TESTS_LABEL in self.labels:
return False
if self.changed_files is None or not self.changed_files:
@ -275,7 +277,7 @@ class PRInfo:
def can_skip_integration_tests(self):
# TODO: See a broken loop
if "force tests" in self.labels:
if FORCE_TESTS_LABEL in self.labels:
return False
if self.changed_files is None or not self.changed_files:
@ -292,7 +294,7 @@ class PRInfo:
def can_skip_functional_tests(self):
# TODO: See a broken loop
if "force tests" in self.labels:
if FORCE_TESTS_LABEL in self.labels:
return False
if self.changed_files is None or not self.changed_files:

29
tests/ci/release.py
View File

@ -9,14 +9,20 @@ import logging
from git_helper import commit, release_branch
from version_helper import (
FILE_WITH_VERSION_PATH,
GENERATED_CONTRIBUTORS,
ClickHouseVersion,
Git,
VersionType,
get_abs_path,
get_version_from_repo,
update_cmake_version,
update_contributors,
)
git = Git()
class Repo:
VALID = ("ssh", "https", "origin")
@ -47,14 +53,16 @@ class Repo:
class Release:
BIG = ("major", "minor")
SMALL = ("patch",)
CMAKE_PATH = get_abs_path(FILE_WITH_VERSION_PATH)
CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS)
def __init__(self, repo: Repo, release_commit: str, release_type: str):
self.repo = repo
self._release_commit = ""
self.release_commit = release_commit
self.release_type = release_type
self._version = get_version_from_repo()
self._git = self._version._git
self._git = git
self._version = get_version_from_repo(git=self._git)
self._release_branch = ""
self._rollback_stack = [] # type: List[str]
@ -75,7 +83,7 @@ class Release:
def read_version(self):
self._git.update()
self.version = get_version_from_repo()
self.version = get_version_from_repo(git=self._git)
def check_prerequisites(self):
"""
@ -179,14 +187,15 @@ class Release:
self.version = self.version.update(self.release_type)
self.version.with_description(version_type)
update_cmake_version(self.version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH)
update_contributors(raise_error=True)
# Checkouting the commit of the branch and not the branch itself,
# then we are able to skip rollback
with self._checkout(f"{self.release_branch}@{{0}}", False):
current_commit = self.run("git rev-parse HEAD")
self.run(
f"git commit -m "
f"'Update version to {self.version.string}' '{cmake_path}'"
f"'Update version to {self.version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
)
with self._push(
"HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch
@ -243,9 +252,10 @@ class Release:
new_version = self.version.patch_update()
new_version.with_description("prestable")
update_cmake_version(new_version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH)
update_contributors(raise_error=True)
self.run(
f"git commit -m 'Update version to {new_version.string}' '{cmake_path}'"
f"git commit -m 'Update version to {new_version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
)
with self._push(self.release_branch):
with self._create_gh_label(
@ -271,9 +281,10 @@ class Release:
self.version = self.version.update(self.release_type)
self.version.with_description("testing")
update_cmake_version(self.version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH)
update_contributors(raise_error=True)
self.run(
f"git commit -m 'Update version to {self.version.string}' '{cmake_path}'"
f"git commit -m 'Update version to {self.version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
)
with self._push(helper_branch):
body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")

3
tests/ci/run_check.py
View File

@ -9,7 +9,7 @@ from github import Github
from commit_status_helper import get_commit, post_labels, remove_labels
from env_helper import GITHUB_RUN_URL, GITHUB_REPOSITORY, GITHUB_SERVER_URL
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from pr_info import FORCE_TESTS_LABEL, PRInfo
from workflow_approve_rerun_lambda.app import TRUSTED_CONTRIBUTORS
NAME = "Run Check (actions)"
@ -23,7 +23,6 @@ TRUSTED_ORG_IDS = {
OK_SKIP_LABELS = {"release", "pr-backport", "pr-cherrypick"}
CAN_BE_TESTED_LABEL = "can be tested"
DO_NOT_TEST_LABEL = "do not test"
FORCE_TESTS_LABEL = "force tests"
SUBMODULE_CHANGED_LABEL = "submodule changed"
LABELS = {

44
tests/ci/version_helper.py
View File

@ -2,9 +2,9 @@
import logging
import os.path as p
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError
from typing import Dict, List, Tuple, Union
from typing import Dict, List, Optional, Tuple, Union
from git_helper import Git, removeprefix
from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix
FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt"
CHANGELOG_IN_PATH = "debian/changelog.in"
@ -34,8 +34,6 @@ SET(VERSION_STRING {string})
# end of autochange
"""
git = Git()
class ClickHouseVersion:
"""Immutable version class. On update returns a new instance"""
@ -46,7 +44,7 @@ class ClickHouseVersion:
minor: Union[int, str],
patch: Union[int, str],
revision: Union[int, str],
git: Git,
git: Optional[Git],
tweak: str = None,
):
self._major = int(major)
@ -54,9 +52,11 @@ class ClickHouseVersion:
self._patch = int(patch)
self._revision = int(revision)
self._git = git
self._tweak = None
self._tweak = TWEAK
if tweak is not None:
self._tweak = int(tweak)
elif self._git is not None:
self._tweak = self._git.tweak
self._describe = ""
def update(self, part: str) -> "ClickHouseVersion":
@ -91,7 +91,7 @@ class ClickHouseVersion:
@property
def tweak(self) -> int:
return self._tweak or self._git.tweak
return self._tweak
@property
def revision(self) -> int:
@ -99,7 +99,9 @@ class ClickHouseVersion:
@property
def githash(self) -> str:
return self._git.sha
if self._git is not None:
return self._git.sha
return "0000000000000000000000000000000000000000"
@property
def describe(self):
@ -171,7 +173,7 @@ def validate_version(version: str):
def get_abs_path(path: str) -> str:
return p.abspath(p.join(git.root, path))
return p.abspath(p.join(git_runner.cwd, path))
def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
@ -197,6 +199,7 @@ def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
def get_version_from_repo(
versions_path: str = FILE_WITH_VERSION_PATH,
git: Optional[Git] = None,
) -> ClickHouseVersion:
versions = read_versions(versions_path)
return ClickHouseVersion(
@ -208,14 +211,16 @@ def get_version_from_repo(
)
def get_version_from_string(version: str) -> ClickHouseVersion:
def get_version_from_string(
version: str, git: Optional[Git] = None
) -> ClickHouseVersion:
validate_version(version)
parts = version.split(".")
return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3])
def get_version_from_tag(tag: str) -> ClickHouseVersion:
git.check_tag(tag)
Git.check_tag(tag)
tag = tag[1:].split("-")[0]
return get_version_from_string(tag)
@ -236,7 +241,7 @@ def version_arg(version: str) -> ClickHouseVersion:
def get_tagged_versions() -> List[ClickHouseVersion]:
versions = []
for tag in git.get_tags():
for tag in get_tags():
try:
version = get_version_from_tag(tag)
versions.append(version)
@ -255,21 +260,26 @@ def update_cmake_version(
def update_contributors(
relative_contributors_path: str = GENERATED_CONTRIBUTORS, force: bool = False
relative_contributors_path: str = GENERATED_CONTRIBUTORS,
force: bool = False,
raise_error: bool = False,
):
# Check if we have shallow checkout by comparing number of lines
# '--is-shallow-repository' is in git since 2.15, 2017-10-30
if git.run("git rev-parse --is-shallow-repository") == "true" and not force:
if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force:
logging.warning("The repository is shallow, refusing to update contributors")
if raise_error:
raise RuntimeError("update_contributors executed on a shallow repository")
return
contributors = git.run("git shortlog HEAD --summary")
# format: " 1016 Alexey Arno"
shortlog = git_runner.run("git shortlog HEAD --summary")
contributors = sorted(
[c.split(maxsplit=1)[-1].replace('"', r"\"") for c in contributors.split("\n")],
[c.split(maxsplit=1)[-1].replace('"', r"\"") for c in shortlog.split("\n")],
)
contributors = [f' "{c}",' for c in contributors]
executer = p.relpath(p.realpath(__file__), git.root)
executer = p.relpath(p.realpath(__file__), git_runner.cwd)
content = CONTRIBUTORS_TEMPLATE.format(
executer=executer, contributors="\n".join(contributors)
)

20
tests/clickhouse-test
View File

@ -356,6 +356,7 @@ class FailureReason(enum.Enum):
RESULT_DIFF = "result differs with reference: "
TOO_LONG = "Test runs too long (> 60s). Make it faster."
INTERNAL_QUERY_FAIL = "Internal query (CREATE/DROP DATABASE) failed:"
LEFT_QUERIES = "Queries left in background after the test finished:"
# SKIPPED reasons
DISABLED = "disabled"
@ -671,6 +672,15 @@ class TestCase:
# We're in Flaky Check mode, check the run time as well while we're at it.
return TestResult(self.name, TestStatus.FAIL, FailureReason.TOO_LONG, total_time, description)
left_queries_check = args.no_left_queries_check is False
if self.tags and 'no-left-queries-check' in self.tags:
left_queries_check = False
if left_queries_check:
processlist = get_processlist_after_test(self.testcase_args)
if processlist:
description += "\n{}\n".format(json.dumps(processlist, indent=4))
return TestResult(self.name, TestStatus.FAIL, FailureReason.LEFT_QUERIES, total_time, description)
if os.path.exists(self.stdout_file):
os.remove(self.stdout_file)
if os.path.exists(self.stderr_file):
@ -747,16 +757,6 @@ class TestCase:
proc.stdout is None or 'Exception' not in proc.stdout)
need_drop_database = not maybe_passed
left_queries_check = args.no_left_queries_check is False
if self.tags and 'no-left-queries-check' in self.tags:
left_queries_check = False
if left_queries_check:
processlist = get_processlist_after_test(args)
if processlist:
print(colored(f"\nFound queries left in processlist after running {args.testcase_basename} (database={database}):", args, "red", attrs=["bold"]))
print(json.dumps(processlist, indent=4))
exit_code.value = 1
if need_drop_database:
seconds_left = max(args.timeout - (datetime.now() - start_time).total_seconds(), 20)
try:

2
tests/queries/0_stateless/00693_max_block_size_system_tables_columns.reference
View File

@ -2,7 +2,7 @@
1
1
1
t_00693 Memory 1 [] 1970-01-01 00:00:00 [] [] CREATE TEMPORARY TABLE t_00693 (`x` UInt8) ENGINE = Memory Memory \N \N
t_00693 Memory 1 [] 1970-01-01 00:00:00 [] [] CREATE TEMPORARY TABLE t_00693 (`x` UInt8) ENGINE = Memory Memory 0 0
1
1
1

2
tests/queries/0_stateless/01085_max_distributed_connections_http.sh
View File

@ -15,3 +15,5 @@ while [[ $i -lt $retries ]]; do
timeout 1.8s ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_distributed_connections=2&max_threads=1" -d "$query" && break
((++i))
done
clickhouse_test_wait_queries 60

78
tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh
View File

@ -10,46 +10,40 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
function thread_create {
while true; do
$CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2"
sleep 0.0$RANDOM
done
function thread_create()
{
$CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2"
sleep 0.0$RANDOM
}
function thread_drop {
while true; do
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1"
sleep 0.0$RANDOM
done
function thread_drop()
{
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1"
sleep 0.0$RANDOM
}
function thread_rename {
while true; do
$CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)'
sleep 0.0$RANDOM
done
function thread_rename()
{
$CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)'
sleep 0.0$RANDOM
}
function thread_select {
while true; do
$CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)'
sleep 0.0$RANDOM
done
function thread_select()
{
$CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)'
sleep 0.0$RANDOM
}
function thread_insert {
while true; do
$CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: '| grep -v -P 'Code: (60|218)'
sleep 0.0$RANDOM
done
function thread_insert()
{
$CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: '| grep -v -P 'Code: (60|218)'
sleep 0.0$RANDOM
}
function thread_insert_select {
while true; do
$CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)'
sleep 0.0$RANDOM
done
function thread_insert_select()
{
$CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)'
sleep 0.0$RANDOM
}
export -f thread_create
@ -65,18 +59,18 @@ export -f thread_insert_select
function test_with_engine {
echo "Testing $1"
timeout 10 bash -c "thread_create t1 $1" &
timeout 10 bash -c "thread_create t2 $1" &
timeout 10 bash -c 'thread_drop t1' &
timeout 10 bash -c 'thread_drop t2' &
timeout 10 bash -c 'thread_rename t1 t2' &
timeout 10 bash -c 'thread_rename t2 t1' &
timeout 10 bash -c 'thread_select t1' &
timeout 10 bash -c 'thread_select t2' &
timeout 10 bash -c 'thread_insert t1 5' &
timeout 10 bash -c 'thread_insert t2 10' &
timeout 10 bash -c 'thread_insert_select t1 t2' &
timeout 10 bash -c 'thread_insert_select t2 t1' &
clickhouse_client_loop_timeout 10 thread_create t1 $1 &
clickhouse_client_loop_timeout 10 thread_create t2 $1 &
clickhouse_client_loop_timeout 10 thread_drop t1 &
clickhouse_client_loop_timeout 10 thread_drop t2 &
clickhouse_client_loop_timeout 10 thread_rename t1 t2 &
clickhouse_client_loop_timeout 10 thread_rename t2 t1 &
clickhouse_client_loop_timeout 10 thread_select t1 &
clickhouse_client_loop_timeout 10 thread_select t2 &
clickhouse_client_loop_timeout 10 thread_insert t1 5 &
clickhouse_client_loop_timeout 10 thread_insert t2 10 &
clickhouse_client_loop_timeout 10 thread_insert_select t1 t2 &
clickhouse_client_loop_timeout 10 thread_insert_select t2 t1 &
wait
echo "Done $1"

6
tests/queries/0_stateless/01675_data_type_coroutine.sh
View File

@ -17,3 +17,9 @@ while [[ $counter -lt $retries ]]; do
done
echo 'Ok'
# wait queries, since there is 'Maximum parse depth' error on the client
# and in this case it simply reset the connection and don't read everything
# from server, so there is no guarantee that the query is stopped when the
# client returns
clickhouse_test_wait_queries 60

6
tests/queries/0_stateless/01675_data_type_coroutine_2.sh
View File

@ -17,3 +17,9 @@ done
#echo "I = ${I}"
echo 'Ok'
# wait queries, since there is 'Maximum parse depth' error on the client
# and in this case it simply reset the connection and don't read everything
# from server, so there is no guarantee that the query is stopped when the
# client returns
clickhouse_test_wait_queries 60

2
tests/queries/0_stateless/01731_async_task_queue_wait.sh
View File

@ -8,3 +8,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# cancellation with async_socket_for_remote=1 (that ignores
# max_distributed_connections)
timeout --signal=SIGINT 1 ${CLICKHOUSE_CLIENT} --max_distributed_connections=1 --max_block_size=2 --interactive_delay=900000 -q "select number + sleep(0.3) as x from remote('127.{2,3}', system.numbers) settings max_block_size = 2" 2>&1 | grep "Empty task was returned from async task queue" || true
clickhouse_test_wait_queries 60

22
tests/queries/0_stateless/02104_overcommit_memory.sh
View File

@ -5,21 +5,19 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -q 'CREATE USER IF NOT EXISTS u1 IDENTIFIED WITH no_password'
$CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u1'
$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u02104'
$CLICKHOUSE_CLIENT -q 'CREATE USER IF NOT EXISTS u02104 IDENTIFIED WITH no_password'
$CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u02104'
function overcommited()
{
while true; do
$CLICKHOUSE_CLIENT -u u1 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_guaranteed_memory_usage=1,memory_usage_overcommit_max_wait_microseconds=500' 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo "OVERCOMMITED WITH USER LIMIT IS KILLED"
done
$CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_guaranteed_memory_usage=1,memory_usage_overcommit_max_wait_microseconds=500' 2>&1 \
| grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo "OVERCOMMITED WITH USER LIMIT IS KILLED"
}
function expect_execution()
{
while true; do
$CLICKHOUSE_CLIENT -u u1 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,max_guaranteed_memory_usage=2,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null
done
$CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,max_guaranteed_memory_usage=2,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null
}
export -f overcommited
@ -29,9 +27,9 @@ function user_test()
{
for _ in {1..10};
do
timeout 10 bash -c overcommited &
timeout 10 bash -c expect_execution &
done;
clickhouse_client_loop_timeout 10 overcommited &
clickhouse_client_loop_timeout 10 expect_execution &
done
wait
}
@ -45,4 +43,4 @@ else
echo "OVERCOMMITED WITH USER LIMIT WAS KILLED"
fi
$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u1'
$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u02104'

7
tests/queries/0_stateless/02151_http_s_structure_set_eof.sh
View File

@ -1,4 +1,5 @@
#!/usr/bin/env bash
# Tags: long
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
@ -25,3 +26,9 @@ timeout 0.15s ${CLICKHOUSE_CURL} -sS -F "s=@$tmp_file;" "${CLICKHOUSE_URL}&s_str
echo $?
timeout 0.15s ${CLICKHOUSE_CURL} -sS -F "s=@$tmp_file;" "${CLICKHOUSE_URL}&s_structure=key+Int&query=SELECT+dummy+IN+s&input_format_parallel_parsing=false" -o /dev/null
echo $?
# wait until the query above will start,
# so that clickhouse_test_wait_queries will see them.
sleep 5
clickhouse_test_wait_queries 60

12
tests/queries/0_stateless/02154_parser_backtracking.sh
View File

@ -12,14 +12,14 @@ echo 'SELECT '"$(perl -e 'print "CAST(" x 100')"'a, b'"$(perl -e 'print ")" x 10
echo 'SELECT '"$(perl -e 'print "CAST(" x 100')"'a AS b'"$(perl -e 'print ")" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'Syntax error'
echo 'SELECT '"$(perl -e 'print "CAST(" x 100')"'1'"$(perl -e 'print ", '"'UInt8'"')" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000
echo 'SELECT '"$(perl -e 'print "CAST(" x 100')"'1'"$(perl -e 'print " AS UInt8)" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000
echo "SELECT fo,22222?LUTAY(SELECT(NOT CAUTAY(SELECT(NOT CAST(NOTT(NOT CAST(NOT NOT LEfT(NOT coARRAYlumnsFLuTAY(SELECT(NO0?LUTAY(SELECT(NOT CAUTAY(SELECT(NOT CAST(NOTT(NOT CAST(NOT NOT LEfT(NOT coARRAYlumnsFLuTAY(SELECT(NOTAYTAY(SELECT(NOTAYEFAULT(fo,22222?LUTAY(%SELECT(NOT CAST(NOT NOTAYTAY(SELECT(NOTAYEFAULT(fo,22222?LUTAY(SELECT(NOT CAST(NOT NOT (NOe)))))))))))))))))))))))))))))))))" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'Syntax error'
echo "SELECT position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(a b))))))))))))))))))))" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'Syntax error'
echo "SELECT position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(a, b))))))))))))))))))))" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'UNKNOWN_IDENTIFIER'
echo "SELECT position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(a, b, c))))))))))))))))))))" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'UNKNOWN_IDENTIFIER'
echo 'SELECT '"$(perl -e 'print "position(" x 100')"'x'"$(perl -e 'print ")" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'UNKNOWN_IDENTIFIER'
echo "SELECT position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(a b))))))))))))))))))))" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'Syntax error'
echo "SELECT position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(a, b))))))))))))))))))))" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'Syntax error'
echo "SELECT position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(position(a, b, c))))))))))))))))))))" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'Syntax error'
echo 'SELECT '"$(perl -e 'print "position(" x 100')"'x'"$(perl -e 'print ")" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'Syntax error'
echo 'SELECT '"$(perl -e 'print "position(" x 100')"'x y'"$(perl -e 'print ")" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'Syntax error'
echo 'SELECT '"$(perl -e 'print "position(" x 100')"'x IN y'"$(perl -e 'print ")" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'UNKNOWN_IDENTIFIER'
echo 'SELECT '"$(perl -e 'print "position(" x 100')"'x IN y'"$(perl -e 'print ")" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'Syntax error'
echo 'SELECT '"$(perl -e 'print "position(" x 100')"'x'"$(perl -e 'print " IN x)" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'UNKNOWN_IDENTIFIER'
echo 'SELECT '"$(perl -e 'print "position(" x 100')"'x'"$(perl -e 'print ", x)" x 100')" | ${CLICKHOUSE_LOCAL} --max_parser_depth 10000 2>&1 | grep -cF 'UNKNOWN_IDENTIFIER'

2
tests/queries/0_stateless/02187_async_inserts_all_formats.reference
View File

@ -14,8 +14,10 @@ JSONCompactStringsEachRow
JSONCompactStringsEachRowWithNames
JSONCompactStringsEachRowWithNamesAndTypes
JSONEachRow
JSONLines
JSONStringsEachRow
MsgPack
NDJSON
Native
ORC
Parquet

32
tests/queries/0_stateless/02267_jsonlines_ndjson_format.reference Normal file
View File

@ -0,0 +1,32 @@
{"number":"0"}
{"number":"1"}
{"number":"2"}
{"number":"3"}
{"number":"4"}
{"number":"5"}
{"number":"6"}
{"number":"7"}
{"number":"8"}
{"number":"9"}
{"number":"0"}
{"number":"1"}
{"number":"2"}
{"number":"3"}
{"number":"4"}
{"number":"5"}
{"number":"6"}
{"number":"7"}
{"number":"8"}
{"number":"9"}
{"n1":1,"n2":2}
{"n1":1,"n2":2}
{"n1":3,"n2":4}
{"n1":3,"n2":4}
{"n1":5,"n2":6}
{"n1":5,"n2":6}
{"n1":1,"n2":2}
{"n1":1,"n2":2}
{"n1":3,"n2":4}
{"n1":3,"n2":4}
{"n1":5,"n2":6}
{"n1":5,"n2":6}

14
tests/queries/0_stateless/02267_jsonlines_ndjson_format.sql Normal file
View File

@ -0,0 +1,14 @@
SELECT * FROM numbers(10) FORMAT JSONLines;
SELECT * FROM numbers(10) FORMAT NDJSON;
DROP TABLE IF EXISTS 02267_t;
CREATE TABLE 02267_t (n1 UInt32, n2 UInt32) ENGINE = Memory;
INSERT INTO 02267_t FORMAT JSONLines {"n1": 1, "n2": 2} {"n1": 3, "n2": 4} {"n1": 5, "n2": 6};
INSERT INTO 02267_t FORMAT NDJSON {"n1": 1, "n2": 2} {"n1": 3, "n2": 4} {"n1": 5, "n2": 6};
SELECT * FROM 02267_t ORDER BY n1, n2 FORMAT JSONLines;
SELECT * FROM 02267_t ORDER BY n1, n2 FORMAT NDJSON;
DROP TABLE 02267_t;

60
tests/queries/0_stateless/02267_special_operator_parse_alias_check.reference Normal file
View File

@ -0,0 +1,60 @@
1234
1234 1234
1234 1234
1234 1234
1234 1234
1234 1234
1234
1234 1234
1234 1234
1234 UInt32
1234 UInt32
1234 1234 UInt32
1234 1234 UInt32
234
234 1234
234 1234
234 2
234 2
234 1234 2
234 1234 2
234 1234 2
23
23 2
23 2
234 1234 2 3
234 1234 2 3
bca a abca
bca a abca
bca abca
bca abca
bca a
bca a
bca
abc a abca
abc a abca
bc a abca
bc a abca
5 2019-05-05
5 2019-05-05
123 1234 123
123 1234 123
1 123 1234
0 123 1234
0 123 1234
2019-05-06 1 2019-05-05
2019-05-06 1 2019-05-05
2019-05-06 1 2019-05-05
2019-05-06 1 2019-05-05
2019-05-04 1 2019-05-05
2019-05-04 1 2019-05-05
2019-05-04 1 2019-05-05
2019-05-04 1 2019-05-05
1 2019-05-05 2019-05-06
1 2019-05-05 2019-05-06
1 2019-05-05 2019-05-06
1 2019-05-05 2019-05-06
1 2019-05-05 2019-05-06
1 2019-05-05 2019-05-06
1 2019-05-05 2019-05-06
1 2019-05-05 2019-05-06

121
tests/queries/0_stateless/02267_special_operator_parse_alias_check.sql Normal file
View File

@ -0,0 +1,121 @@
-- CAST expression
-- cast(expr [[AS] alias_1] AS Type)
SELECT cast('1234' AS UInt32);
SELECT cast('1234' AS lhs AS UInt32), lhs;
SELECT cast('1234' lhs AS UInt32), lhs;
SELECT cast(('1234' AS lhs) AS UInt32), lhs;
SELECT cast(('1234' AS lhs) rhs AS UInt32), rhs;
SELECT cast(('1234' AS lhs) AS rhs AS UInt32), rhs;
-- cast(expr [[AS] alias_1], type_expr [[as] alias_2])
SELECT cast('1234', 'UInt32');
SELECT cast('1234' AS lhs, 'UInt32'), lhs;
SELECT cast('1234' lhs, 'UInt32'), lhs;
SELECT cast('1234', 'UInt32' AS rhs), rhs;
SELECT cast('1234', 'UInt32' rhs), rhs;
SELECT cast('1234' AS lhs, 'UInt32' AS rhs), lhs, rhs;
SELECT cast('1234' lhs, 'UInt32' rhs), lhs, rhs;
-- SUBSTRING expression
-- SUBSTRING(expr FROM start)
SELECT substring('1234' FROM 2);
SELECT substring('1234' AS lhs FROM 2), lhs;
SELECT substring('1234' lhs FROM 2), lhs;
SELECT substring('1234' FROM 2 AS rhs), rhs;
SELECT substring('1234' FROM 2 rhs), rhs;
SELECT substring('1234' AS lhs FROM 2 AS rhs), lhs, rhs;
SELECT substring('1234' lhs FROM 2 rhs), lhs, rhs;
SELECT substring(('1234' AS lhs) FROM (2 AS rhs)), lhs, rhs;
-- SUBSTRING(expr FROM start FOR length)
SELECT substring('1234' FROM 2 FOR 2);
SELECT substring('1234' FROM 2 FOR 2 AS lhs), lhs;
SELECT substring('1234' FROM 2 FOR 2 lhs), lhs;
-- SUBSTRING(expr, start, length)
SELECT substring('1234' AS arg_1, 2 AS arg_2, 3 AS arg_3), arg_1, arg_2, arg_3;
SELECT substring('1234' arg_1, 2 arg_2, 3 arg_3), arg_1, arg_2, arg_3;
-- -- TRIM expression ([[LEADING|TRAILING|BOTH] trim_character FROM] input_string)
SELECT trim(LEADING 'a' AS arg_1 FROM 'abca' AS arg_2), arg_1, arg_2;
SELECT trim(LEADING 'a' arg_1 FROM 'abca' arg_2), arg_1, arg_2;
SELECT trim(LEADING 'a' FROM 'abca' AS arg_2), arg_2;
SELECT trim(LEADING 'a' FROM 'abca' arg_2), arg_2;
SELECT trim(LEADING 'a' AS arg_1 FROM 'abca'), arg_1;
SELECT trim(LEADING 'a' arg_1 FROM 'abca'), arg_1;
SELECT trim(LEADING 'a' FROM 'abca');
SELECT trim(TRAILING 'a' AS arg_1 FROM 'abca' AS arg_2), arg_1, arg_2;
SELECT trim(TRAILING 'a' arg_1 FROM 'abca' arg_2), arg_1, arg_2;
SELECT trim(BOTH 'a' AS arg_1 FROM 'abca' AS arg_2), arg_1, arg_2;
SELECT trim(BOTH 'a' arg_1 FROM 'abca' arg_2), arg_1, arg_2;
-- EXTRACT expression
-- EXTRACT(part FROM date)
SELECT EXTRACT(DAY FROM toDate('2019-05-05') as arg_1), arg_1;
SELECT EXTRACT(DAY FROM toDate('2019-05-05') arg_1), arg_1;
-- Function extract(haystack, pattern)
SELECT extract('1234' AS arg_1, '123' AS arg_2), arg_1, arg_2;
SELECT extract('1234' arg_1, '123' arg_2), arg_1, arg_2;
-- POSITION expression
-- position(needle IN haystack)
SELECT position(('123' AS arg_1) IN ('1234' AS arg_2)), arg_1, arg_2;
-- position(haystack, needle[, start_pos])
SELECT position('123' AS arg_1, '1234' AS arg_2), arg_1, arg_2;
SELECT position('123' arg_1, '1234' arg_2), arg_1, arg_2;
-- dateAdd, dateSub expressions
-- function(unit, offset, timestamp)
SELECT dateAdd(DAY, 1 AS arg_1, toDate('2019-05-05') AS arg_2), arg_1, arg_2;
SELECT dateAdd(DAY, 1 arg_1, toDate('2019-05-05') arg_2), arg_1, arg_2;
-- function(offset, timestamp)
SELECT dateAdd(DAY, 1 AS arg_1, toDate('2019-05-05') AS arg_2), arg_1, arg_2;
SELECT dateAdd(DAY, 1 arg_1, toDate('2019-05-05') arg_2), arg_1, arg_2;
-- function(unit, offset, timestamp)
SELECT dateSub(DAY, 1 AS arg_1, toDate('2019-05-05') AS arg_2), arg_1, arg_2;
SELECT dateSub(DAY, 1 arg_1, toDate('2019-05-05') arg_2), arg_1, arg_2;
-- function(offset, timestamp)
SELECT dateSub(DAY, 1 AS arg_1, toDate('2019-05-05') AS arg_2), arg_1, arg_2;
SELECT dateSub(DAY, 1 arg_1, toDate('2019-05-05') arg_2), arg_1, arg_2;
-- dateDiff expression
-- dateDiff(unit, startdate, enddate, [timezone])
SELECT dateDiff(DAY, toDate('2019-05-05') AS arg_1, toDate('2019-05-06') AS arg_2), arg_1, arg_2;
SELECT dateDiff(DAY, toDate('2019-05-05') arg_1, toDate('2019-05-06') arg_2), arg_1, arg_2;
SELECT dateDiff(DAY, toDate('2019-05-05') AS arg_1, toDate('2019-05-06') AS arg_2, 'UTC'), arg_1, arg_2;
SELECT dateDiff(DAY, toDate('2019-05-05') arg_1, toDate('2019-05-06') arg_2, 'UTC'), arg_1, arg_2;
-- dateDiff('unit', startdate, enddate, [timezone])
SELECT dateDiff('DAY', toDate('2019-05-05') AS arg_1, toDate('2019-05-06') AS arg_2), arg_1, arg_2;
SELECT dateDiff('DAY', toDate('2019-05-05') arg_1, toDate('2019-05-06') arg_2), arg_1, arg_2;
SELECT dateDiff('DAY', toDate('2019-05-05') AS arg_1, toDate('2019-05-06') AS arg_2, 'UTC'), arg_1, arg_2;
SELECT dateDiff('DAY', toDate('2019-05-05') arg_1, toDate('2019-05-06') arg_2, 'UTC'), arg_1, arg_2;

6
tests/queries/0_stateless/02270_errors_in_files.reference Normal file
View File

@ -0,0 +1,6 @@
test_02270_2.csv
test_02270_2.csv
test_02270_2.csv
test_02270_2.csv
test_02270_2.csv.gz
test_02270_2.csv.gz

35
tests/queries/0_stateless/02270_errors_in_files.sh Executable file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env bash
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
[ -e "${CLICKHOUSE_TMP}"/test_02270_1.csv ] && rm "${CLICKHOUSE_TMP}"/test_02270_1.csv
[ -e "${CLICKHOUSE_TMP}"/test_02270_2.csv ] && rm "${CLICKHOUSE_TMP}"/test_02270_2.csv
echo "Hello,World" > "${CLICKHOUSE_TMP}"/test_02270_1.csv
echo "Error" > "${CLICKHOUSE_TMP}"/test_02270_2.csv
${CLICKHOUSE_LOCAL} --query "SELECT * FROM file('${CLICKHOUSE_TMP}/test_02270*.csv', CSV, 'a String, b String')" 2>&1 | grep -o "test_02270_2.csv"
${CLICKHOUSE_LOCAL} --query "SELECT * FROM file('${CLICKHOUSE_TMP}/test_02270*.csv', CSV, 'a String, b String')" --input_format_parallel_parsing 0 2>&1 | grep -o "test_02270_2.csv"
user_files_path=$($CLICKHOUSE_CLIENT --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep -E '^Code: 107.*FILE_DOESNT_EXIST' | head -1 | awk '{gsub("/nonexist.txt","",$9); print $9}')
${CLICKHOUSE_CLIENT} --query "INSERT INTO TABLE FUNCTION file('test_02270_1.csv') SELECT 'Hello', 'World'"
${CLICKHOUSE_CLIENT} --query "INSERT INTO TABLE FUNCTION file('test_02270_2.csv') SELECT 'Error'"
${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('test_02270*.csv', 'CSV', 'a String, b String')" 2>&1 | grep -o -m1 "test_02270_2.csv"
${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('test_02270*.csv', 'CSV', 'a String, b String')" --input_format_parallel_parsing 0 2>&1 | grep -o -m1 "test_02270_2.csv"
${CLICKHOUSE_CLIENT} --query "INSERT INTO TABLE FUNCTION file('test_02270_1.csv.gz') SELECT 'Hello', 'World'"
${CLICKHOUSE_CLIENT} --query "INSERT INTO TABLE FUNCTION file('test_02270_2.csv.gz') SELECT 'Error'"
${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('test_02270*.csv.gz', 'CSV', 'a String, b String')" 2>&1 | grep -o -m1 "test_02270_2.csv.gz"
${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('test_02270*.csv.gz', 'CSV', 'a String, b String')" --input_format_parallel_parsing 0 2>&1 | grep -o -m1 "test_02270_2.csv.gz"
rm "${CLICKHOUSE_TMP}"/test_02270_1.csv
rm "${CLICKHOUSE_TMP}"/test_02270_2.csv
rm "${user_files_path}"/test_02270_1.csv
rm "${user_files_path}"/test_02270_2.csv
rm "${user_files_path}"/test_02270_1.csv.gz
rm "${user_files_path}"/test_02270_2.csv.gz

2
tests/queries/0_stateless/02270_errors_in_files_s3.reference Normal file
View File

@ -0,0 +1,2 @@
http://localhost:11111/test/tsv_with_header.tsv
test/tsv_with_header.tsv

9
tests/queries/0_stateless/02270_errors_in_files_s3.sh Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
# Tags: no-fasttest
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('http://localhost:11111/test/{a,tsv_with_header}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64')" 2>&1 | grep -o -m1 "http://localhost:11111/test/tsv_with_header.tsv"
${CLICKHOUSE_CLIENT} --query "SELECT * FROM s3('http://localhost:11111/test/{a,tsv_with_header}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64')" 2>&1 | grep -o -m1 "test/tsv_with_header.tsv"

10
tests/queries/0_stateless/02271_int_sql_compatibility.reference Normal file
View File

@ -0,0 +1,10 @@
CREATE TEMPORARY TABLE t1_02271\n(\n `x` Int32\n)\nENGINE = Memory
CREATE TEMPORARY TABLE t2_02271\n(\n `x` Int32 DEFAULT 1\n)\nENGINE = Memory
CREATE TEMPORARY TABLE t3_02271\n(\n `x` UInt32\n)\nENGINE = Memory
CREATE TEMPORARY TABLE t4_02271\n(\n `x` Int32\n)\nENGINE = Memory
CREATE TEMPORARY TABLE t5_02271\n(\n `x` Int32 DEFAULT 1\n)\nENGINE = Memory
CREATE TEMPORARY TABLE t6_02271\n(\n `x` Int32\n)\nENGINE = Memory
CREATE TEMPORARY TABLE t7_02271\n(\n `x` Int32 DEFAULT 1\n)\nENGINE = Memory
CREATE TEMPORARY TABLE t8_02271\n(\n `x` UInt32\n)\nENGINE = Memory
CREATE TEMPORARY TABLE t9_02271\n(\n `x` Int32\n)\nENGINE = Memory
CREATE TEMPORARY TABLE t10_02271\n(\n `x` Int32 DEFAULT 1\n)\nENGINE = Memory

29
tests/queries/0_stateless/02271_int_sql_compatibility.sql Normal file
View File

@ -0,0 +1,29 @@
CREATE TEMPORARY TABLE t1_02271 (x INT(11));
SHOW CREATE TEMPORARY TABLE t1_02271;
CREATE TEMPORARY TABLE t2_02271 (x INT(11) DEFAULT 1);
SHOW CREATE TEMPORARY TABLE t2_02271;
CREATE TEMPORARY TABLE t3_02271 (x INT(11) UNSIGNED);
SHOW CREATE TEMPORARY TABLE t3_02271;
CREATE TEMPORARY TABLE t4_02271 (x INT(11) SIGNED);
SHOW CREATE TEMPORARY TABLE t4_02271;
CREATE TEMPORARY TABLE t5_02271 (x INT(11) SIGNED DEFAULT 1);
SHOW CREATE TEMPORARY TABLE t5_02271;
CREATE TEMPORARY TABLE t6_02271 (x INT());
SHOW CREATE TEMPORARY TABLE t6_02271;
CREATE TEMPORARY TABLE t7_02271 (x INT() DEFAULT 1);
SHOW CREATE TEMPORARY TABLE t7_02271;
CREATE TEMPORARY TABLE t8_02271 (x INT() UNSIGNED);
SHOW CREATE TEMPORARY TABLE t8_02271;
CREATE TEMPORARY TABLE t9_02271 (x INT() SIGNED);
SHOW CREATE TEMPORARY TABLE t9_02271;
CREATE TEMPORARY TABLE t10_02271 (x INT() SIGNED DEFAULT 1);
SHOW CREATE TEMPORARY TABLE t10_02271;

1
tests/queries/0_stateless/02271_temporary_table_show_rows_bytes.reference Normal file
View File

@ -0,0 +1 @@
02271_temporary_table_show_rows_bytes 1000 8192

4
tests/queries/0_stateless/02271_temporary_table_show_rows_bytes.sql Normal file
View File

@ -0,0 +1,4 @@
-- NOTE: database = currentDatabase() is not mandatory
CREATE TEMPORARY TABLE 02271_temporary_table_show_rows_bytes (A Int64) Engine=Memory as SELECT * FROM numbers(1000);
SELECT database, name, total_rows, total_bytes FROM system.tables WHERE is_temporary AND name = '02271_temporary_table_show_rows_bytes';

2
tests/queries/1_stateful/00168_parallel_processing_on_replicas_part_1.sh
View File

@ -10,7 +10,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# All replicas are localhost, disable `prefer_localhost_replica` option to test network interface
# Currently this feature could not work with hedged requests
# Enabling `enable_sample_offset_parallel_processing` feature could lead to intersecting marks, so some of them would be thrown away and it will lead to incorrect result of SELECT query
SETTINGS="--max_parallel_replicas=3 --prefer_localhost_replica=false --use_hedged_requests=false --async_socket_for_remote=false --allow_experimental_parallel_reading_from_replicas=true"
SETTINGS="--max_parallel_replicas=3 --use_hedged_requests=false --async_socket_for_remote=false --allow_experimental_parallel_reading_from_replicas=true"
# Prepare tables
$CLICKHOUSE_CLIENT $SETTINGS -nm -q '''

1
utils/list-versions/version_date.tsv
View File

@ -1,3 +1,4 @@
v22.4.2.1-stable 2022-04-22
v22.3.3.44-lts 2022-04-06
v22.3.2.2-lts 2022-03-17
v22.2.3.5-stable 2022-02-25

1 v22.3.3.44-lts v22.4.2.1-stable 2022-04-06 2022-04-22
1 v22.4.2.1-stable 2022-04-22
2 v22.3.3.44-lts v22.3.3.44-lts 2022-04-06 2022-04-06
3 v22.3.2.2-lts v22.3.2.2-lts 2022-03-17 2022-03-17
4 v22.2.3.5-stable v22.2.3.5-stable 2022-02-25 2022-02-25