mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 09:32:06 +00:00
Merge branch 'master' into bash-complete-log-level
This commit is contained in:
mergify[bot]
GitHub
commit
5bcba95e72
79
.github/workflows/codeql.yml
vendored
Normal file
79
.github/workflows/codeql.yml
vendored
Normal file
@ -0,0 +1,79 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
"on":
|
||||
# push:
|
||||
# branches: [ master ]
|
||||
# pull_request:
|
||||
# # The branches below must be a subset of the branches above
|
||||
# branches: [ master ]
|
||||
schedule:
|
||||
- cron: '0 */6 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
CC: clang-14
|
||||
CXX: clang++-14
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: ['cpp']
|
||||
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
|
||||
# Learn more about CodeQL language support at https://git.io/codeql-language-support
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
submodules: 'true'
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
# - name: Autobuild
|
||||
# uses: github/codeql-action/autobuild@v2
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
sudo apt-get install -yq git cmake python ninja-build
|
||||
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 ..
|
||||
ninja
|
||||
rm -rf ../contrib
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v2
|
||||
32
.github/workflows/docs_check.yml
vendored
32
.github/workflows/docs_check.yml
vendored
@ -92,6 +92,38 @@ jobs:
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/changed_images.json
|
||||
StyleCheck:
|
||||
needs: DockerHubPush
|
||||
runs-on: [self-hosted, style-checker]
|
||||
if: ${{ success() || failure() }}
|
||||
steps:
|
||||
- name: Set envs
|
||||
run: |
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
TEMP_PATH=${{ runner.temp }}/style_check
|
||||
EOF
|
||||
- name: Download changed images
|
||||
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ env.TEMP_PATH }}
|
||||
- name: Clear repository
|
||||
run: |
|
||||
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Style Check
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 style_check.py
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker kill "$(docker ps -q)" ||:
|
||||
docker rm -f "$(docker ps -a -q)" ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
DocsCheck:
|
||||
needs: DockerHubPush
|
||||
runs-on: [self-hosted, func-tester-aarch64]
|
||||
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@ -33,10 +33,6 @@
|
||||
/docs/zh/single.md
|
||||
/docs/ja/single.md
|
||||
/docs/fa/single.md
|
||||
/docs/en/development/cmake-in-clickhouse.md
|
||||
/docs/ja/development/cmake-in-clickhouse.md
|
||||
/docs/zh/development/cmake-in-clickhouse.md
|
||||
/docs/ru/development/cmake-in-clickhouse.md
|
||||
|
||||
# callgrind files
|
||||
callgrind.out.*
|
||||
|
||||
@ -211,13 +211,8 @@ function run_tests
|
||||
if [ -v CHPC_TEST_RUN_BY_HASH_TOTAL ]; then
|
||||
# filter tests array in bash https://stackoverflow.com/a/40375567
|
||||
for index in "${!test_files[@]}"; do
|
||||
# sorry for this, just calculating hash(test_name) % total_tests_group == my_test_group_num
|
||||
test_hash_result=$(echo test_files[$index] | perl -ne 'use Digest::MD5 qw(md5); print unpack('Q', md5($_)) % $ENV{CHPC_TEST_RUN_BY_HASH_TOTAL} == $ENV{CHPC_TEST_RUN_BY_HASH_NUM};')
|
||||
# BTW, for some reason when hash(test_name) % total_tests_group != my_test_group_num perl outputs nothing, not zero
|
||||
if [ "$test_hash_result" != "1" ]; then
|
||||
# deleting element from array
|
||||
[ $(( index % CHPC_TEST_RUN_BY_HASH_TOTAL )) != "$CHPC_TEST_RUN_BY_HASH_NUM" ] && \
|
||||
unset -v 'test_files[$index]'
|
||||
fi
|
||||
done
|
||||
# to have sequential indexes...
|
||||
test_files=("${test_files[@]}")
|
||||
|
||||
550
docs/en/development/cmake-in-clickhouse.md
Normal file
550
docs/en/development/cmake-in-clickhouse.md
Normal file
@ -0,0 +1,550 @@
|
||||
---
|
||||
sidebar_position: 69
|
||||
sidebar_label: CMake in ClickHouse
|
||||
description: How to make ClickHouse compile and link faster
|
||||
---
|
||||
|
||||
# CMake in ClickHouse
|
||||
|
||||
How to make ClickHouse compile and link faster. Minimal ClickHouse build example:
|
||||
|
||||
```bash
|
||||
cmake .. \
|
||||
-DCMAKE_C_COMPILER=$(which clang-13) \
|
||||
-DCMAKE_CXX_COMPILER=$(which clang++-13) \
|
||||
-DCMAKE_BUILD_TYPE=Debug \
|
||||
-DENABLE_CLICKHOUSE_ALL=OFF \
|
||||
-DENABLE_CLICKHOUSE_SERVER=ON \
|
||||
-DENABLE_CLICKHOUSE_CLIENT=ON \
|
||||
-DENABLE_LIBRARIES=OFF \
|
||||
-DUSE_UNWIND=ON \
|
||||
-DENABLE_UTILS=OFF \
|
||||
-DENABLE_TESTS=OFF
|
||||
```
|
||||
|
||||
## CMake files types
|
||||
|
||||
1. ClickHouse's source CMake files (located in the root directory and in /src).
|
||||
2. Arch-dependent CMake files (located in /cmake/*os_name*).
|
||||
3. Libraries finders (search for contrib libraries, located in /contrib/*/CMakeLists.txt).
|
||||
4. Contrib build CMake files (used instead of libraries' own CMake files, located in /cmake/modules)
|
||||
|
||||
## List of CMake flags
|
||||
- The flag name is a link to its position in the code.
|
||||
- If an option's default value is itself an option, it's also a link to its position in this list.
|
||||
|
||||
## ClickHouse modes
|
||||
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Default value</th>
|
||||
<th>Description</th>
|
||||
<th>Comment</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-all"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Enable all ClickHouse modes by default</td>
|
||||
<td>The <code class="syntax">clickhouse</code> binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-benchmark"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L20" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_BENCHMARK</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Queries benchmarking mode</td>
|
||||
<td><a href="https://clickhouse.com/docs/en/operations/utilities/clickhouse-benchmark/" target="_blank">https://clickhouse.com/docs/en/operations/utilities/clickhouse-benchmark/</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-client"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L13" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_CLIENT</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Client mode (interactive tui/shell that connects to the server)</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-compressor"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L25" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_COMPRESSOR</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Data compressor and decompressor</td>
|
||||
<td><a href="https://clickhouse.com/docs/en/operations/utilities/clickhouse-compressor/" target="_blank">https://clickhouse.com/docs/en/operations/utilities/clickhouse-compressor/</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-copier"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L28" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_COPIER</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Inter-cluster data copying mode</td>
|
||||
<td><a href="https://clickhouse.com/docs/en/operations/utilities/clickhouse-copier/" target="_blank">https://clickhouse.com/docs/en/operations/utilities/clickhouse-copier/</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-extract-from-config"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L22" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Configs processor (extract values etc.)</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-format"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L30" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_FORMAT</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Queries pretty-printer and formatter with syntax highlighting</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-git-import"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L50" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_GIT_IMPORT</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>A tool to analyze Git repositories</td>
|
||||
<td><a href="https://presentations.clickhouse.com/matemarketing_2020/" target="_blank">https://presentations.clickhouse.com/matemarketing_2020/</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-install"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L67" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_INSTALL</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-keeper"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L54" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_KEEPER</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>ClickHouse alternative to ZooKeeper</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-keeper-converter"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L56" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_KEEPER_CONVERTER</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Util allows to convert ZooKeeper logs and snapshots into clickhouse-keeper snapshot</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-library-bridge"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L46" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_LIBRARY_BRIDGE</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>HTTP-server working like a proxy to Library dictionary source</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-local"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L17" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_LOCAL</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Local files fast processing mode</td>
|
||||
<td><a href="https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/" target="_blank">https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-obfuscator"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L34" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_OBFUSCATOR</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Table data obfuscator (convert real data to benchmark-ready one)</td>
|
||||
<td><a href="https://clickhouse.com/docs/en/operations/utilities/clickhouse-obfuscator/" target="_blank">https://clickhouse.com/docs/en/operations/utilities/clickhouse-obfuscator/</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-odbc-bridge"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L40" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_ODBC_BRIDGE</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>HTTP-server working like a proxy to ODBC driver</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-server"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_SERVER</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>Server mode (main mode)</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clickhouse-static-files-disk-uploader"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L52" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CLICKHOUSE_ALL</code></td>
|
||||
<td>A tool to export table data files to be later put to a static files web server</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
|
||||
## External libraries
|
||||
Note that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Default value</th>
|
||||
<th>Description</th>
|
||||
<th>Comment</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><a name="enable-avx"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L19" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_AVX</code></a></td>
|
||||
<td><code class="syntax">0</code></td>
|
||||
<td>Use AVX instructions on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-avx"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L20" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_AVX2</code></a></td>
|
||||
<td><code class="syntax">0</code></td>
|
||||
<td>Use AVX2 instructions on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-avx-for-spec-op"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L23" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_AVX2_FOR_SPEC_OP</code></a></td>
|
||||
<td><code class="syntax">0</code></td>
|
||||
<td>Use avx2 instructions for specific operations on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-avx"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L21" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_AVX512</code></a></td>
|
||||
<td><code class="syntax">0</code></td>
|
||||
<td>Use AVX512 instructions on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-avx-for-spec-op"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L24" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_AVX512_FOR_SPEC_OP</code></a></td>
|
||||
<td><code class="syntax">0</code></td>
|
||||
<td>Use avx512 instructions for specific operations on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-bmi"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L22" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_BMI</code></a></td>
|
||||
<td><code class="syntax">0</code></td>
|
||||
<td>Use BMI instructions on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-ccache"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/ccache.cmake#L22" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CCACHE</code></a></td>
|
||||
<td><code class="syntax">ENABLE_CCACHE_BY_DEFAULT</code></td>
|
||||
<td>Speedup re-compilations using ccache (external tool)</td>
|
||||
<td><a href="https://ccache.dev/" target="_blank">https://ccache.dev/</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-clang-tidy"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L2" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CLANG_TIDY</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Use clang-tidy static analyzer</td>
|
||||
<td><a href="https://clang.llvm.org/extra/clang-tidy/" target="_blank">https://clang.llvm.org/extra/clang-tidy/</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-pclmulqdq"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L17" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_PCLMULQDQ</code></a></td>
|
||||
<td><code class="syntax">1</code></td>
|
||||
<td>Use pclmulqdq instructions on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-popcnt"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L18" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_POPCNT</code></a></td>
|
||||
<td><code class="syntax">1</code></td>
|
||||
<td>Use popcnt instructions on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-sse"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L15" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_SSE41</code></a></td>
|
||||
<td><code class="syntax">1</code></td>
|
||||
<td>Use SSE4.1 instructions on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-sse"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L16" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_SSE42</code></a></td>
|
||||
<td><code class="syntax">1</code></td>
|
||||
<td>Use SSE4.2 instructions on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-ssse"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L14" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_SSSE3</code></a></td>
|
||||
<td><code class="syntax">1</code></td>
|
||||
<td>Use SSSE3 instructions on x86_64</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
|
||||
## Other flags
|
||||
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Default value</th>
|
||||
<th>Description</th>
|
||||
<th>Comment</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><a name="add-gdb-index-for-gold"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L226" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ADD_GDB_INDEX_FOR_GOLD</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Add .gdb-index to resulting binaries for gold linker.</td>
|
||||
<td>Ignored if <code class="syntax">lld</code> is used</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="arch-native"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/cpu_features.cmake#L26" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ARCH_NATIVE</code></a></td>
|
||||
<td><code class="syntax">0</code></td>
|
||||
<td>Add -march=native compiler flag. This makes your binaries non-portable but more performant code may be generated. This option overrides ENABLE_* options for specific instruction set. Highly not recommended to use.</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="build-standalone-keeper"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L253" rel="external nofollow noreferrer" target="_blank"><code class="syntax">BUILD_STANDALONE_KEEPER</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Build keeper as small standalone binary</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="clickhouse-split-binary"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L113" rel="external nofollow noreferrer" target="_blank"><code class="syntax">CLICKHOUSE_SPLIT_BINARY</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="compiler-pipe"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L281" rel="external nofollow noreferrer" target="_blank"><code class="syntax">COMPILER_PIPE</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>-pipe compiler option</td>
|
||||
<td>Less <code class="syntax">/tmp</code> usage, more RAM usage.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-build-path-mapping"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L299" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_BUILD_PATH_MAPPING</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Enable remap file source paths in debug info, predefined preprocessor macros and __builtin_FILE(). It's to generate reproducible builds. See <a href="https://reproducible-builds.org/docs/build-path" target="_blank">https://reproducible-builds.org/docs/build-path</a></td>
|
||||
<td>Reproducible builds If turned <code class="syntax">ON</code>, remap file source paths in debug info, predefined preprocessor macros and __builtin_FILE().</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-check-heavy-builds"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L81" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_CHECK_HEAVY_BUILDS</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Don't allow C++ translation units to compile too long or to take too much memory while compiling.</td>
|
||||
<td>Take care to add prlimit in command line before ccache, or else ccache thinks that prlimit is compiler, and clang++ is its input file, and refuses to work with multiple inputs, e.g in ccache log: [2021-03-31T18:06:32.655327 36900] Command line: /usr/bin/ccache prlimit --as=10000000000 --data=5000000000 --cpu=600 /usr/bin/clang++-11 - ...... std=gnu++2a -MD -MT src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -MF src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o.d -o src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -c ../src/Storages/MergeTree/IMergeTreeDataPart.cpp [2021-03-31T18:06:32.656704 36900] Multiple input files: /usr/bin/clang++-11 and ../src/Storages/MergeTree/IMergeTreeDataPart.cpp Another way would be to use --ccache-skip option before clang++-11 to make ccache ignore it.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-examples"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L201" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_EXAMPLES</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Build all example programs in 'examples' subdirectories</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-fuzzing"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L129" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_FUZZING</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Fuzzy testing using libfuzzer</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-libraries"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L413" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_LIBRARIES</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Enable all external libraries by default</td>
|
||||
<td>Turns on all external libs like s3, kafka, ODBC, ...</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-multitarget-code"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L102" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_MULTITARGET_CODE</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Enable platform-dependent code</td>
|
||||
<td>ClickHouse developers may use platform-dependent code under some macro (e.g. <code class="syntax">ifdef ENABLE_MULTITARGET</code>). If turned ON, this option defines such macro. See <code class="syntax">src/Functions/TargetSpecific.h</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-tests"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L200" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_TESTS</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Provide unit_test_dbms target with Google.Test unit tests</td>
|
||||
<td>If turned <code class="syntax">ON</code>, assumes the user has either the system GTest library or the bundled one.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="enable-thinlto"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L386" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_THINLTO</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Clang-specific link time optimization</td>
|
||||
<td><a href="https://clang.llvm.org/docs/ThinLTO.html" target="_blank">https://clang.llvm.org/docs/ThinLTO.html</a> Applies to clang only. Disabled when building with tests or sanitizers.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="fail-on-unsupported-options-combination"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L32" rel="external nofollow noreferrer" target="_blank"><code class="syntax">FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy</td>
|
||||
<td>If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="glibc-compatibility"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L205" rel="external nofollow noreferrer" target="_blank"><code class="syntax">GLIBC_COMPATIBILITY</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Enable compatibility with older glibc libraries.</td>
|
||||
<td>Only for Linux, x86_64 or aarch64.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="install-stripped-binaries"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L270" rel="external nofollow noreferrer" target="_blank"><code class="syntax">INSTALL_STRIPPED_BINARIES</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Build stripped binaries with debug info in separate directory</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="linker-name"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L58" rel="external nofollow noreferrer" target="_blank"><code class="syntax">LINKER_NAME</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Linker name or full path</td>
|
||||
<td>Example values: <code class="syntax">lld-10</code>, <code class="syntax">gold</code>.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="parallel-compile-jobs"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L10" rel="external nofollow noreferrer" target="_blank"><code class="syntax">PARALLEL_COMPILE_JOBS</code></a></td>
|
||||
<td><code class="syntax">""</code></td>
|
||||
<td>Maximum number of concurrent compilation jobs</td>
|
||||
<td>1 if not set</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="parallel-link-jobs"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L13" rel="external nofollow noreferrer" target="_blank"><code class="syntax">PARALLEL_LINK_JOBS</code></a></td>
|
||||
<td><code class="syntax">""</code></td>
|
||||
<td>Maximum number of concurrent link jobs</td>
|
||||
<td>1 if not set</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="sanitize"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L7" rel="external nofollow noreferrer" target="_blank"><code class="syntax">SANITIZE</code></a></td>
|
||||
<td><code class="syntax">""</code></td>
|
||||
<td>Enable one of the code sanitizers</td>
|
||||
<td>Possible values: - <code class="syntax">address</code> (ASan) - <code class="syntax">memory</code> (MSan) - <code class="syntax">thread</code> (TSan) - <code class="syntax">undefined</code> (UBSan) - "" (no sanitizing)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="split-shared-libraries"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L111" rel="external nofollow noreferrer" target="_blank"><code class="syntax">SPLIT_SHARED_LIBRARIES</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Keep all internal libraries as separate .so files</td>
|
||||
<td>DEVELOPER ONLY. Faster linking if turned on.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="strip-debug-symbols-functions"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L48" rel="external nofollow noreferrer" target="_blank"><code class="syntax">STRIP_DEBUG_SYMBOLS_FUNCTIONS</code></a></td>
|
||||
<td><code class="syntax">STRIP_DSF_DEFAULT</code></td>
|
||||
<td>Do not generate debugger info for ClickHouse functions</td>
|
||||
<td>Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)."</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="use-debug-helpers"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L252" rel="external nofollow noreferrer" target="_blank"><code class="syntax">USE_DEBUG_HELPERS</code></a></td>
|
||||
<td><code class="syntax">USE_DEBUG_HELPERS</code></td>
|
||||
<td>Enable debug helpers</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="use-static-libraries"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L106" rel="external nofollow noreferrer" target="_blank"><code class="syntax">USE_STATIC_LIBRARIES</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Disable to use shared libraries</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="use-unwind"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/unwind.cmake#L1" rel="external nofollow noreferrer" target="_blank"><code class="syntax">USE_UNWIND</code></a></td>
|
||||
<td><code class="syntax">ENABLE_LIBRARIES</code></td>
|
||||
<td>Enable libunwind (better stacktraces)</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="werror"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L417" rel="external nofollow noreferrer" target="_blank"><code class="syntax">WERROR</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Enable -Werror compiler option</td>
|
||||
<td>Using system libs can cause a lot of warnings in includes (on macro expansion).</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="weverything"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L15" rel="external nofollow noreferrer" target="_blank"><code class="syntax">WEVERYTHING</code></a></td>
|
||||
<td><code class="syntax">ON</code></td>
|
||||
<td>Enable -Weverything option with some exceptions.</td>
|
||||
<td>Add some warnings that are not available even with -Wall -Wextra -Wpedantic. Intended for exploration of new compiler warnings that may be found useful. Applies to clang only</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a name="with-coverage"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L344" rel="external nofollow noreferrer" target="_blank"><code class="syntax">WITH_COVERAGE</code></a></td>
|
||||
<td><code class="syntax">OFF</code></td>
|
||||
<td>Profile the resulting binary/binaries</td>
|
||||
<td>Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
## Developer's guide for adding new CMake options
|
||||
|
||||
#### Don't be obvious. Be informative.
|
||||
|
||||
Bad:
|
||||
|
||||
```
|
||||
option (ENABLE_TESTS "Enables testing" OFF)
|
||||
```
|
||||
|
||||
This description is quite useless as it neither gives the viewer any additional information nor explains the option purpose.
|
||||
|
||||
Better:
|
||||
|
||||
```
|
||||
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF)
|
||||
```
|
||||
|
||||
If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some
|
||||
pre-conditions, leave a comment above the option() line and explain what it does.
|
||||
The best way would be linking the docs page (if it exists).
|
||||
The comment is parsed into a separate column (see below).
|
||||
|
||||
Even better:
|
||||
|
||||
```
|
||||
# implies ${TESTS_ARE_ENABLED}
|
||||
# see tests/CMakeLists.txt for implementation detail.
|
||||
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF)
|
||||
```
|
||||
|
||||
#### If the option's state could produce unwanted (or unusual) result, explicitly warn the user.
|
||||
|
||||
Suppose you have an option that may strip debug symbols from the ClickHouse's part.
|
||||
This can speed up the linking process, but produces a binary that cannot be debugged.
|
||||
In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong.
|
||||
Also, such options should be disabled if applies.
|
||||
|
||||
Bad:
|
||||
|
||||
```
|
||||
option(STRIP_DEBUG_SYMBOLS_FUNCTIONS
|
||||
"Do not generate debugger info for ClickHouse functions.
|
||||
${STRIP_DSF_DEFAULT})
|
||||
|
||||
if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
|
||||
target_compile_options(clickhouse_functions PRIVATE "-g0")
|
||||
endif()
|
||||
```
|
||||
|
||||
Better:
|
||||
|
||||
```
|
||||
# Provides faster linking and lower binary size.
|
||||
# Tradeoff is the inability to debug some source files with e.g. gdb
|
||||
# (empty stack frames and no local variables)."
|
||||
option(STRIP_DEBUG_SYMBOLS_FUNCTIONS
|
||||
"Do not generate debugger info for ClickHouse functions."
|
||||
${STRIP_DSF_DEFAULT})
|
||||
|
||||
if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
|
||||
message(WARNING "Not generating debugger info for ClickHouse functions")
|
||||
target_compile_options(clickhouse_functions PRIVATE "-g0")
|
||||
endif()
|
||||
```
|
||||
|
||||
#### In the option's description, explain WHAT the option does rather than WHY it does something.
|
||||
The WHY explanation should be placed in the comment. You may find that the option's name is self-descriptive.
|
||||
|
||||
Bad:
|
||||
|
||||
```
|
||||
option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON)
|
||||
```
|
||||
|
||||
Better:
|
||||
|
||||
```
|
||||
# Only applicable for clang.
|
||||
# Turned off when building with tests or sanitizers.
|
||||
option(ENABLE_THINLTO "Clang-specific link time optimisation" ON).
|
||||
```
|
||||
|
||||
#### Don't assume other developers know as much as you do.
|
||||
In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to
|
||||
the tool's docs. It won't take much of your time.
|
||||
|
||||
Bad:
|
||||
|
||||
```
|
||||
option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON)
|
||||
```
|
||||
|
||||
Better (combined with the above hint):
|
||||
|
||||
```
|
||||
# https://clang.llvm.org/docs/ThinLTO.html
|
||||
# Only applicable for clang.
|
||||
# Turned off when building with tests or sanitizers.
|
||||
option(ENABLE_THINLTO "Clang-specific link time optimisation" ON).
|
||||
```
|
||||
|
||||
Other example, bad:
|
||||
|
||||
```
|
||||
option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF)
|
||||
```
|
||||
|
||||
Better:
|
||||
|
||||
```
|
||||
# https://github.com/include-what-you-use/include-what-you-use
|
||||
option (USE_INCLUDE_WHAT_YOU_USE "Reduce unneeded #include s (external tool)" OFF)
|
||||
```
|
||||
|
||||
#### Prefer consistent default values.
|
||||
CMake allows you to pass a plethora of values representing boolean true/false, e.g. 1, ON, YES, ....
|
||||
|
||||
Prefer the ON/OFF values, if possible.
|
||||
|
||||
62
docs/en/development/integrating_rust_libraries.md
Normal file
62
docs/en/development/integrating_rust_libraries.md
Normal file
@ -0,0 +1,62 @@
|
||||
# Integrating Rust libraries into ClickHouse.
|
||||
|
||||
Rust library integration will be described based on BLAKE3 hash-function integration.
|
||||
|
||||
The first step is forking a library and making neccessary changes for Rust and C/C++ compatibility.
|
||||
|
||||
After forking library repository you need to change target settings in Cargo.toml file. Firstly, you need to switch build to static library. Secondly, you need to add cbindgen crate to the crate list. We will use it later to generate C-header automatically.
|
||||
|
||||
The next step is creating or editing the build.rs script for your library - and enable cbindgen to generate the header during library build. These lines were added to BLAKE3 build script for the same purpose:
|
||||
|
||||
```
|
||||
let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||
|
||||
let package_name = env::var("CARGO_PKG_NAME").unwrap();
|
||||
let output_file = ("include/".to_owned() + &format!("{}.h", package_name)).to_string();
|
||||
|
||||
match cbindgen::generate(&crate_dir) {
|
||||
Ok(header) => {
|
||||
header.write_to_file(&output_file);
|
||||
}
|
||||
Err(err) => {
|
||||
panic!("{}", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
As you can see, script sets the output directory and launches header generation.
|
||||
|
||||
The next step is to add CMake files into library directory, so it can build with other submodules. As you can see, BLAKE3 main directory contains two CMake files - CMakeLists.txt and build_rust_lib.cmake. The second one is a function, which calls cargo build and sets all needed paths for library build. You should copy it to your library and then you can adjust cargo flags and other settings for you library needs.
|
||||
|
||||
When finished with CMake configuration, you should move on to create a C/C++ compatible API for your library. Let us see BLAKE3's method blake3_apply_shim:
|
||||
|
||||
```
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn blake3_apply_shim(
|
||||
begin: *const c_char,
|
||||
_size: u32,
|
||||
out_char_data: *mut u8,
|
||||
) -> *mut c_char {
|
||||
if begin.is_null() {
|
||||
let err_str = CString::new("input was a null pointer").unwrap();
|
||||
return err_str.into_raw();
|
||||
}
|
||||
let mut hasher = Hasher::new();
|
||||
let input_bytes = CStr::from_ptr(begin);
|
||||
let input_res = input_bytes.to_bytes();
|
||||
hasher.update(input_res);
|
||||
let mut reader = hasher.finalize_xof();
|
||||
reader.fill(std::slice::from_raw_parts_mut(out_char_data, OUT_LEN));
|
||||
std::ptr::null_mut()
|
||||
}
|
||||
```
|
||||
|
||||
This method gets C-compatible string, its size and output string pointer as input. Then, it converts C-compatible inputs into types that are used by actual library methods and calls them. After that, it should convert library methods' outputs back into C-compatible type. In that particular case library supported direct writing into pointer by method fill(), so the convertion was not needed. The main advice here is to create less methods, so you will need to do less convertions on each method call and won't create much overhead.
|
||||
|
||||
Also, you should use attribute #[no_mangle] and extern "C" for every C-compatible attribute. Without it library can compile incorrectly and cbindgen won't launch header autogeneration.
|
||||
|
||||
After all these steps you can test your library in a small project to find all problems with compatibility or header generation. If any problems occur during header generation, you can try to configure it with cbindgen.toml file (you can find an example of it in BLAKE3 directory or a template here: [https://github.com/eqrion/cbindgen/blob/master/template.toml](https://github.com/eqrion/cbindgen/blob/master/template.toml)). If everything works correctly, you can finally integrate its methods into ClickHouse.
|
||||
|
||||
In addition, some problems with integration are worth noting here:
|
||||
1) Some architectures may require special cargo flags or build.rs configurations, so you may want to test cross-compilation for different platforms first.
|
||||
2) MemorySanitizer can cause false-positive reports as it's unable to see if some variables in Rust are initialized or not. It was solved with writing a method with more explicit definition for some variables, although this implementation of method is slower and is used only to fix MemorySanitizer builds.
|
||||
@ -544,6 +544,7 @@ Keys:
|
||||
- `errorlog` – Error log file.
|
||||
- `size` – Size of the file. Applies to `log` and `errorlog`. Once the file reaches `size`, ClickHouse archives and renames it, and creates a new log file in its place.
|
||||
- `count` – The number of archived log files that ClickHouse stores.
|
||||
- `console` – Send `log` and `errorlog` to the console instead of file. To enable, set to `1` or `true`.
|
||||
|
||||
**Example**
|
||||
|
||||
@ -557,6 +558,15 @@ Keys:
|
||||
</logger>
|
||||
```
|
||||
|
||||
Writing to the console can be configured. Config example:
|
||||
|
||||
``` xml
|
||||
<logger>
|
||||
<level>information</level>
|
||||
<console>1</console>
|
||||
</logger>
|
||||
```
|
||||
|
||||
Writing to the syslog is also supported. Config example:
|
||||
|
||||
``` xml
|
||||
@ -677,8 +687,8 @@ On hosts with low RAM and swap, you possibly need setting `max_server_memory_usa
|
||||
|
||||
## max_concurrent_queries {#max-concurrent-queries}
|
||||
|
||||
The maximum number of simultaneously processed queries related to MergeTree table.
|
||||
Queries may be limited by other settings: [max_concurrent_insert_queries](#max-concurrent-insert-queries), [max_concurrent_select_queries](#max-concurrent-select-queries), [max_concurrent_queries_for_user](#max-concurrent-queries-for-user), [max_concurrent_queries_for_all_users](#max-concurrent-queries-for-all-users), [min_marks_to_honor_max_concurrent_queries](#min-marks-to-honor-max-concurrent-queries).
|
||||
The maximum number of simultaneously processed queries.
|
||||
Note that other limits also apply: [max_concurrent_insert_queries](#max-concurrent-insert-queries), [max_concurrent_select_queries](#max-concurrent-select-queries), [max_concurrent_queries_for_user](#max-concurrent-queries-for-user), [max_concurrent_queries_for_all_users](#max-concurrent-queries-for-all-users).
|
||||
|
||||
:::note
|
||||
These settings can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
|
||||
@ -694,7 +704,7 @@ Default value: `100`.
|
||||
**Example**
|
||||
|
||||
``` xml
|
||||
<max_concurrent_queries>100</max_concurrent_queries>
|
||||
<max_concurrent_queries>200</max_concurrent_queries>
|
||||
```
|
||||
|
||||
## max_concurrent_insert_queries {#max-concurrent-insert-queries}
|
||||
@ -781,21 +791,6 @@ Default value: `0`.
|
||||
|
||||
- [max_concurrent_queries](#max-concurrent-queries)
|
||||
|
||||
## min_marks_to_honor_max_concurrent_queries {#min-marks-to-honor-max-concurrent-queries}
|
||||
|
||||
The minimal number of marks read by the query for applying the [max_concurrent_queries](#max-concurrent-queries) setting.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Disabled.
|
||||
|
||||
**Example**
|
||||
|
||||
``` xml
|
||||
<min_marks_to_honor_max_concurrent_queries>10</min_marks_to_honor_max_concurrent_queries>
|
||||
```
|
||||
|
||||
## max_connections {#max-connections}
|
||||
|
||||
The maximum number of inbound connections.
|
||||
|
||||
@ -381,3 +381,36 @@ Possible values:
|
||||
|
||||
Default value: `1` second.
|
||||
|
||||
## max_concurrent_queries {#max-concurrent-queries}
|
||||
|
||||
Max number of concurrently executed queries related to the MergeTree table. Queries will still be limited by other `max_concurrent_queries` settings.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — No limit.
|
||||
|
||||
Default value: `0` (no limit).
|
||||
|
||||
**Example**
|
||||
|
||||
``` xml
|
||||
<max_concurrent_queries>50</max_concurrent_queries>
|
||||
```
|
||||
|
||||
## min_marks_to_honor_max_concurrent_queries {#min-marks-to-honor-max-concurrent-queries}
|
||||
|
||||
The minimal number of marks read by the query for applying the [max_concurrent_queries](#max-concurrent-queries) setting. Note that queries will still be limited by other `max_concurrent_queries` settings.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Disabled (`max_concurrent_queries` limit applied to no queries).
|
||||
|
||||
Default value: `0` (limit never applied).
|
||||
|
||||
**Example**
|
||||
|
||||
``` xml
|
||||
<min_marks_to_honor_max_concurrent_queries>10</min_marks_to_honor_max_concurrent_queries>
|
||||
```
|
||||
|
||||
@ -17,18 +17,18 @@ SELECT * FROM system.metrics LIMIT 10
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─metric─────────────────────┬─value─┬─description──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ Query │ 1 │ Number of executing queries │
|
||||
│ Merge │ 0 │ Number of executing background merges │
|
||||
│ PartMutation │ 0 │ Number of mutations (ALTER DELETE/UPDATE) │
|
||||
│ ReplicatedFetch │ 0 │ Number of data parts being fetched from replicas │
|
||||
│ ReplicatedSend │ 0 │ Number of data parts being sent to replicas │
|
||||
│ ReplicatedChecks │ 0 │ Number of data parts checking for consistency │
|
||||
│ BackgroundPoolTask │ 0 │ Number of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping) │
|
||||
│ BackgroundSchedulePoolTask │ 0 │ Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc. │
|
||||
│ DiskSpaceReservedForMerge │ 0 │ Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts. │
|
||||
│ DistributedSend │ 0 │ Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode. │
|
||||
└────────────────────────────┴───────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
┌─metric───────────────────────────────┬─value─┬─description────────────────────────────────────────────────────────────┐
|
||||
│ Query │ 1 │ Number of executing queries │
|
||||
│ Merge │ 0 │ Number of executing background merges │
|
||||
│ PartMutation │ 0 │ Number of mutations (ALTER DELETE/UPDATE) │
|
||||
│ ReplicatedFetch │ 0 │ Number of data parts being fetched from replicas │
|
||||
│ ReplicatedSend │ 0 │ Number of data parts being sent to replicas │
|
||||
│ ReplicatedChecks │ 0 │ Number of data parts checking for consistency │
|
||||
│ BackgroundMergesAndMutationsPoolTask │ 0 │ Number of active merges and mutations in an associated background pool │
|
||||
│ BackgroundFetchesPoolTask │ 0 │ Number of active fetches in an associated background pool │
|
||||
│ BackgroundCommonPoolTask │ 0 │ Number of active tasks in an associated background pool │
|
||||
│ BackgroundMovePoolTask │ 0 │ Number of active tasks in BackgroundProcessingPool for moves │
|
||||
└──────────────────────────────────────┴───────┴────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
@ -408,7 +408,7 @@ SELECT toTypeName(JSON_VALUE('{"hello":2}', '$.hello'));
|
||||
Result:
|
||||
|
||||
``` text
|
||||
"world"
|
||||
world
|
||||
0
|
||||
2
|
||||
String
|
||||
|
||||
62
docs/ru/development/integrating_rust_libraries.md
Normal file
62
docs/ru/development/integrating_rust_libraries.md
Normal file
@ -0,0 +1,62 @@
|
||||
# Интеграция библиотек на языке Rust в ClickHouse.
|
||||
|
||||
Интеграция библиотек будет описываться на основе работы проведенной для библиотеки BLAKE3.
|
||||
|
||||
Первым шагом интеграции является создание форка библиотеки для внесения дальнейших изменений по совместимости методов на Rust с C/C++.
|
||||
|
||||
В форке необходимо будет изменить конфигурацию Cargo.toml, сменив таргет на статическую библиотеку. Кроме того, необходимо добавить crate cbindgen для его дальнейшего использования при сборке.
|
||||
|
||||
Необходимо создать либо отредактировать сборочный скрипт build.rs, добавив в него запуск cbindgen - автогенератора заголовочных файлов .h. Пример такого запуска можно увидеть в build.rs для BLAKE3:
|
||||
|
||||
```
|
||||
let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||
|
||||
let package_name = env::var("CARGO_PKG_NAME").unwrap();
|
||||
let output_file = ("include/".to_owned() + &format!("{}.h", package_name)).to_string();
|
||||
|
||||
match cbindgen::generate(&crate_dir) {
|
||||
Ok(header) => {
|
||||
header.write_to_file(&output_file);
|
||||
}
|
||||
Err(err) => {
|
||||
panic!("{}", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Скрипт назначает директорию для создания залоговочного файла и в конце запускает метод генерации cbindgen.
|
||||
|
||||
Далее необходимо подключить библиотеку к CMake. В BLAKE3 для этого были созданы два файла - CMakeLists.txt и файл, содержащий функцию для запуска cargo build как таргета, - build_rust_lib.cmake. Последний стоит скопировать в подключаемую библиотеку и отредактировать в соотвествии с требуемыми параметрами для сборки - добавить флаги или какие-либо настройки для разных архитектур.
|
||||
|
||||
Завершив настройку CMake, можно приступить к созданию методов-прослоек, которые обеспечат совместимость библиотеки и остального кода ClickHouse. В частности, рассмотрим такой метод, написанный для BLAKE3:
|
||||
|
||||
```
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn blake3_apply_shim(
|
||||
begin: *const c_char,
|
||||
_size: u32,
|
||||
out_char_data: *mut u8,
|
||||
) -> *mut c_char {
|
||||
if begin.is_null() {
|
||||
let err_str = CString::new("input was a null pointer").unwrap();
|
||||
return err_str.into_raw();
|
||||
}
|
||||
let mut hasher = Hasher::new();
|
||||
let input_bytes = CStr::from_ptr(begin);
|
||||
let input_res = input_bytes.to_bytes();
|
||||
hasher.update(input_res);
|
||||
let mut reader = hasher.finalize_xof();
|
||||
reader.fill(std::slice::from_raw_parts_mut(out_char_data, OUT_LEN));
|
||||
std::ptr::null_mut()
|
||||
}
|
||||
```
|
||||
|
||||
На вход метод принимает строку в C-совместимом формате, её размер и указатель, в который будет положен результат. Кроме того, для того, чтобы иметь возможность вывести ошибку, метод возвращает строку с ней как результат работы (и нулевой указатель в случае отсутствия ошибок). C-совместимые не используются в методах BLAKE3, поэтому они конвертируются посредством соотвествующих структур и методов в привычные форматы для языка Rust. Далее запускаются оригинальные методы библиотеки. Их результат следует преобразовать обратно в C-совместимые структуры, однако в данном случае удается избежать обратной конвертации, поскольку библиотека поддерживает запись напрямую по указателю *mut u8.
|
||||
|
||||
Кроме того, стоит отметить обязательность аттрибута #[no_mangle] и указания extern "C" для всех таких методов. Без них не удастся провести корректную совместимую с C/C++ компиляцию и автогенерацию заголовков.
|
||||
|
||||
После этих действий можно протестировать компиляцию и работу методов на небольшом проекте для выявляения несовместимостей и ошибок. Если возникают пробемы с генерацией заголовков, может потребоваться поработать с конфигурацией cbindgen через файл cbindgen.toml, найти который можно либо в BLAKE3, либо взяв оригинальный темплейт разработчика cbindgen: [https://github.com/eqrion/cbindgen/blob/master/template.toml](https://github.com/eqrion/cbindgen/blob/master/template.toml).
|
||||
|
||||
В заключение, стоит отметить пару пробелм, возникших при интеграции BLAKE3:
|
||||
1) Некоторые архитектуры могут потребовать настройки компиляции в build.rs и в build_rust_lib.cmake в связи со своими особенностями.
|
||||
2) MemorySanitizer плохо понимает инициализацию памяти в Rust, поэтому для избежания ложноположительных срабатываний для BLAKE3 был создан альтернативный метод, который более явно, но при этом медленнее, инициализировал память. Он компилируется только для сборки с MemorySanitizer и в релиз не попадает. Вероятно, возможны и более красивые способы решения этой проблемы, но при интеграции BLAKE3 они не были обнаружены.
|
||||
@ -17,18 +17,18 @@ SELECT * FROM system.metrics LIMIT 10
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─metric─────────────────────┬─value─┬─description──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ Query │ 1 │ Number of executing queries │
|
||||
│ Merge │ 0 │ Number of executing background merges │
|
||||
│ PartMutation │ 0 │ Number of mutations (ALTER DELETE/UPDATE) │
|
||||
│ ReplicatedFetch │ 0 │ Number of data parts being fetched from replicas │
|
||||
│ ReplicatedSend │ 0 │ Number of data parts being sent to replicas │
|
||||
│ ReplicatedChecks │ 0 │ Number of data parts checking for consistency │
|
||||
│ BackgroundPoolTask │ 0 │ Number of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping) │
|
||||
│ BackgroundSchedulePoolTask │ 0 │ Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc. │
|
||||
│ DiskSpaceReservedForMerge │ 0 │ Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts. │
|
||||
│ DistributedSend │ 0 │ Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode. │
|
||||
└────────────────────────────┴───────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
┌─metric───────────────────────────────┬─value─┬─description────────────────────────────────────────────────────────────┐
|
||||
│ Query │ 1 │ Number of executing queries │
|
||||
│ Merge │ 0 │ Number of executing background merges │
|
||||
│ PartMutation │ 0 │ Number of mutations (ALTER DELETE/UPDATE) │
|
||||
│ ReplicatedFetch │ 0 │ Number of data parts being fetched from replicas │
|
||||
│ ReplicatedSend │ 0 │ Number of data parts being sent to replicas │
|
||||
│ ReplicatedChecks │ 0 │ Number of data parts checking for consistency │
|
||||
│ BackgroundMergesAndMutationsPoolTask │ 0 │ Number of active merges and mutations in an associated background pool │
|
||||
│ BackgroundFetchesPoolTask │ 0 │ Number of active fetches in an associated background pool │
|
||||
│ BackgroundCommonPoolTask │ 0 │ Number of active tasks in an associated background pool │
|
||||
│ BackgroundMovePoolTask │ 0 │ Number of active tasks in BackgroundProcessingPool for moves │
|
||||
└──────────────────────────────────────┴───────┴────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Смотрите также**
|
||||
|
||||
@ -407,7 +407,7 @@ SELECT toTypeName(JSON_VALUE('{"hello":2}', '$.hello'));
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
"world"
|
||||
world
|
||||
0
|
||||
2
|
||||
String
|
||||
|
||||
@ -8,9 +8,9 @@ sidebar_label: "\u5982\u4F55\u5728Linux\u4E0A\u6784\u5EFAClickHouse for AARCH64\
|
||||
|
||||
# 如何在Linux上为AARCH64（ARM64）架构构建ClickHouse {#how-to-build-clickhouse-on-linux-for-aarch64-arm64-architecture}
|
||||
|
||||
这是当你有Linux机器，并希望使用它来构建的情况下 `clickhouse` 二进制文件将运行在另一个Linux机器上与AARCH64CPU架构。 这适用于在Linux服务器上运行的持续集成检查。
|
||||
有种情况，当你有台Linux机器，并希望用它来构建 `clickhouse` 的二进制文件，将其运行在另一台AARCH64CPU架构的Linux机器上。这适用于在Linux服务器上运行的持续集成检查。
|
||||
|
||||
Aarch64的交叉构建基于 [构建说明](../development/build.md) 先跟着他们
|
||||
Aarch64的交叉构建基于 [构建说明](../development/build.md) 让我们先睹为快。
|
||||
|
||||
# 安装Clang-8 {#install-clang-8}
|
||||
|
||||
@ -32,7 +32,7 @@ wget 'https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel
|
||||
tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build-aarch64/cmake/toolchain/linux-aarch64 --strip-components=1
|
||||
```
|
||||
|
||||
# 建立ClickHouse {#build-clickhouse}
|
||||
# 构建ClickHouse {#build-clickhouse}
|
||||
|
||||
``` bash
|
||||
cd ClickHouse
|
||||
@ -41,4 +41,4 @@ CC=clang-8 CXX=clang++-8 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linu
|
||||
ninja -C build-arm64
|
||||
```
|
||||
|
||||
生成的二进制文件将仅在具有AARCH64CPU体系结构的Linux上运行。
|
||||
生成的二进制文件将仅在AARCH64CPU体系架构的Linux上运行。
|
||||
|
||||
@ -522,7 +522,7 @@ ClickHouse 在数据片段合并时会删除掉过期的数据。
|
||||
|
||||
当ClickHouse发现数据过期时, 它将会执行一个计划外的合并。要控制这类合并的频率, 您可以设置 `merge_with_ttl_timeout`。如果该值被设置的太低, 它将引发大量计划外的合并，这可能会消耗大量资源。
|
||||
|
||||
如果在合并的过程中执行 `SELECT` 查询, 则可能会得到过期的数据。为了避免这种情况，可以在 `SELECT` 之前使用 [OPTIMIZE](../../../engines/table-engines/mergetree-family/mergetree.md#misc_operations-optimize) 。
|
||||
如果在两次合并的时间间隔中执行 `SELECT` 查询, 则可能会得到过期的数据。为了避免这种情况，可以在 `SELECT` 之前使用 [OPTIMIZE](../../../engines/table-engines/mergetree-family/mergetree.md#misc_operations-optimize) 。
|
||||
|
||||
## 使用多个块设备进行数据存储 {#table_engine-mergetree-multiple-volumes}
|
||||
|
||||
|
||||
@ -5,8 +5,9 @@ sidebar_label: "ClickHouse用户"
|
||||
|
||||
# ClickHouse用户 {#clickhouse-adopters}
|
||||
|
||||
!!! warning "免责声明"
|
||||
如下使用ClickHouse的公司和他们的成功案例来源于公开资源，因此和实际情况可能有所出入。如果您分享您公司使用ClickHouse的故事，我们将不胜感激 [将其添加到列表](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/introduction/adopters.md)，但请确保你这样做不会有任何保密协议的问题。也欢迎提供来自其他公司的出版物的更新。
|
||||
:::note 免责声明
|
||||
如下使用ClickHouse的公司和他们的成功案例来源于公开资源，因此和实际情况可能有所出入。如果您分享您公司使用ClickHouse的故事，我们将不胜感激 [将其添加到列表](https://github.com/ClickHouse/ClickHouse/edit/master/docs/zh/introduction/adopters.md)，但请确保你这样做不会有任何保密协议的问题。也欢迎提供来自其他公司的出版物的更新。
|
||||
:::
|
||||
|
||||
| 公司简介 | 行业 | 用例 | 群集大小 | (Un)压缩数据大小<abbr title="of single replica"><sup>\*</sup></abbr> | 参考资料 |
|
||||
|-----------------------------------------------------------------|------------------|----------------|---------------------------------------------------|----------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
---
|
||||
sidebar_label: 设置
|
||||
sidebar_position: 51
|
||||
slug: /zh/operations/settings/
|
||||
---
|
||||
|
||||
# 设置 {#settings}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
sidebar_label: 设置
|
||||
sidebar_position: 52
|
||||
slug: /zh/operations/settings/settings
|
||||
---
|
||||
|
||||
# 设置 {#settings}
|
||||
|
||||
@ -17,18 +17,18 @@ SELECT * FROM system.metrics LIMIT 10
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─metric─────────────────────┬─value─┬─description──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ Query │ 1 │ Number of executing queries │
|
||||
│ Merge │ 0 │ Number of executing background merges │
|
||||
│ PartMutation │ 0 │ Number of mutations (ALTER DELETE/UPDATE) │
|
||||
│ ReplicatedFetch │ 0 │ Number of data parts being fetched from replicas │
|
||||
│ ReplicatedSend │ 0 │ Number of data parts being sent to replicas │
|
||||
│ ReplicatedChecks │ 0 │ Number of data parts checking for consistency │
|
||||
│ BackgroundPoolTask │ 0 │ Number of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping) │
|
||||
│ BackgroundSchedulePoolTask │ 0 │ Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc. │
|
||||
│ DiskSpaceReservedForMerge │ 0 │ Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts. │
|
||||
│ DistributedSend │ 0 │ Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode. │
|
||||
└────────────────────────────┴───────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
┌─metric───────────────────────────────┬─value─┬─description────────────────────────────────────────────────────────────┐
|
||||
│ Query │ 1 │ Number of executing queries │
|
||||
│ Merge │ 0 │ Number of executing background merges │
|
||||
│ PartMutation │ 0 │ Number of mutations (ALTER DELETE/UPDATE) │
|
||||
│ ReplicatedFetch │ 0 │ Number of data parts being fetched from replicas │
|
||||
│ ReplicatedSend │ 0 │ Number of data parts being sent to replicas │
|
||||
│ ReplicatedChecks │ 0 │ Number of data parts checking for consistency │
|
||||
│ BackgroundMergesAndMutationsPoolTask │ 0 │ Number of active merges and mutations in an associated background pool │
|
||||
│ BackgroundFetchesPoolTask │ 0 │ Number of active fetches in an associated background pool │
|
||||
│ BackgroundCommonPoolTask │ 0 │ Number of active tasks in an associated background pool │
|
||||
│ BackgroundMovePoolTask │ 0 │ Number of active tasks in BackgroundProcessingPool for moves │
|
||||
└──────────────────────────────────────┴───────┴────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**另请参阅**
|
||||
|
||||
@ -1 +0,0 @@
|
||||
../../../en/sql-reference/data-types/map.md
|
||||
107
docs/zh/sql-reference/data-types/map.md
Normal file
107
docs/zh/sql-reference/data-types/map.md
Normal file
@ -0,0 +1,107 @@
|
||||
---
|
||||
sidebar_position: 65
|
||||
sidebar_label: Map(key, value)
|
||||
---
|
||||
|
||||
# Map(key, value) {#data_type-map}
|
||||
|
||||
`Map(key, value)` 可以存储 `key:value` 键值对类型的数据。
|
||||
|
||||
**参数**
|
||||
|
||||
- `key` — 键值对的key，类型可以是：[String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md), [LowCardinality](../../sql-reference/data-types/lowcardinality.md), 或者 [FixedString](../../sql-reference/data-types/fixedstring.md).
|
||||
- `value` — 键值对的value，类型可以是：[String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md), [Array](../../sql-reference/data-types/array.md), [LowCardinality](../../sql-reference/data-types/lowcardinality.md), 或者 [FixedString](../../sql-reference/data-types/fixedstring.md).
|
||||
|
||||
使用 `a['key']` 可以从 `a Map('key', 'value')` 类型的列中获取到对应的值，这是一个线性复杂度的查询。
|
||||
|
||||
**示例**
|
||||
|
||||
示例表:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory;
|
||||
INSERT INTO table_map VALUES ({'key1':1, 'key2':10}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
|
||||
```
|
||||
|
||||
查询 `key2` 的所有值:
|
||||
|
||||
```sql
|
||||
SELECT a['key2'] FROM table_map;
|
||||
```
|
||||
查询结果:
|
||||
|
||||
```text
|
||||
┌─arrayElement(a, 'key2')─┐
|
||||
│ 10 │
|
||||
│ 20 │
|
||||
│ 30 │
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
如果在 `Map()` 类型的列中，查询的 `key` 值不存在，那么根据 `value` 的类型，查询结果将会是数字0，空字符串或者空数组。
|
||||
|
||||
```sql
|
||||
INSERT INTO table_map VALUES ({'key3':100}), ({});
|
||||
SELECT a['key3'] FROM table_map;
|
||||
```
|
||||
|
||||
查询结果:
|
||||
|
||||
```text
|
||||
┌─arrayElement(a, 'key3')─┐
|
||||
│ 100 │
|
||||
│ 0 │
|
||||
└─────────────────────────┘
|
||||
┌─arrayElement(a, 'key3')─┐
|
||||
│ 0 │
|
||||
│ 0 │
|
||||
│ 0 │
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
## 将Tuple类型转换成Map类型 {#map-and-tuple}
|
||||
|
||||
您可以使用 [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) 方法将 `Tuple()` 转换成 `Map()` ：
|
||||
|
||||
``` sql
|
||||
SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─map───────────────────────────┐
|
||||
│ {1:'Ready',2:'Steady',3:'Go'} │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
## Map.keys 和 Map.values 单独使用 {#map-subcolumns}
|
||||
|
||||
为了更好使用 `Map` 类型，在一定的场景下，可以单独使用 `keys` 或者 `values`，而不需要将整个列的数据都读取出来。
|
||||
|
||||
**示例**
|
||||
|
||||
查询:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE t_map (`a` Map(String, UInt64)) ENGINE = Memory;
|
||||
INSERT INTO t_map VALUES (map('key1', 1, 'key2', 2, 'key3', 3));
|
||||
SELECT a.keys FROM t_map;
|
||||
SELECT a.values FROM t_map;
|
||||
```
|
||||
|
||||
结果:
|
||||
|
||||
``` text
|
||||
┌─a.keys─────────────────┐
|
||||
│ ['key1','key2','key3'] │
|
||||
└────────────────────────┘
|
||||
┌─a.values─┐
|
||||
│ [1,2,3] │
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
**另请参阅**
|
||||
|
||||
- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function
|
||||
- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function
|
||||
|
||||
[Original article](https://clickhouse.com/docs/zh/sql-reference/data-types/map/) <!--hide-->
|
||||
@ -1 +0,0 @@
|
||||
../../../../en/sql-reference/statements/alter/setting.md
|
||||
57
docs/zh/sql-reference/statements/alter/setting.md
Normal file
57
docs/zh/sql-reference/statements/alter/setting.md
Normal file
@ -0,0 +1,57 @@
|
||||
---
|
||||
sidebar_position: 38
|
||||
sidebar_label: SETTING
|
||||
---
|
||||
|
||||
# 表设置操作 {#table_settings_manipulations}
|
||||
|
||||
这是一组更改表设置的操作。你可以修改设置或将其重置为默认值。单个查询可以同时更改多个设置。 如果指定名称的设置不存在，则查询会引发异常。
|
||||
|
||||
**语法**
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY|RESET SETTING ...
|
||||
```
|
||||
|
||||
!!! note "注意"
|
||||
这些查询只能应用于 [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) 表。
|
||||
|
||||
|
||||
## 修改设置 {#alter_modify_setting}
|
||||
|
||||
更改表设置
|
||||
|
||||
**语法**
|
||||
|
||||
```sql
|
||||
MODIFY SETTING setting_name=value [, ...]
|
||||
```
|
||||
|
||||
**示例**
|
||||
|
||||
```sql
|
||||
CREATE TABLE example_table (id UInt32, data String) ENGINE=MergeTree() ORDER BY id;
|
||||
ALTER TABLE example_table MODIFY SETTING max_part_loading_threads=8, max_parts_in_total=50000;
|
||||
```
|
||||
|
||||
## 重置设置 {#alter_reset_setting}
|
||||
|
||||
重置表设置为默认值。如果设置处于默认状态，则不采取任何操作。
|
||||
|
||||
**语法**
|
||||
|
||||
```sql
|
||||
RESET SETTING setting_name [, ...]
|
||||
```
|
||||
|
||||
**示例**
|
||||
|
||||
```sql
|
||||
CREATE TABLE example_table (id UInt32, data String) ENGINE=MergeTree() ORDER BY id
|
||||
SETTINGS max_part_loading_threads=8;
|
||||
ALTER TABLE example_table RESET SETTING max_part_loading_threads;
|
||||
```
|
||||
|
||||
**参见**
|
||||
|
||||
- [MergeTree settings](../../../operations/settings/merge-tree-settings.md)
|
||||
@ -1507,8 +1507,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
}
|
||||
|
||||
#if defined(OS_LINUX)
|
||||
auto tasks_stats_provider = TasksStatsCounters::findBestAvailableProvider();
|
||||
if (tasks_stats_provider == TasksStatsCounters::MetricsProvider::None)
|
||||
if (!TasksStatsCounters::checkIfAvailable())
|
||||
{
|
||||
LOG_INFO(log, "It looks like this system does not have procfs mounted at /proc location,"
|
||||
" neither clickhouse-server process has CAP_NET_ADMIN capability."
|
||||
@ -1519,10 +1518,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
" It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers.",
|
||||
executable_path);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_INFO(log, "Tasks stats provider: {}", TasksStatsCounters::metricsProviderString(tasks_stats_provider));
|
||||
}
|
||||
|
||||
if (!hasLinuxCapability(CAP_SYS_NICE))
|
||||
{
|
||||
|
||||
@ -16,6 +16,13 @@
|
||||
#include <Interpreters/Access/InterpreterGrantQuery.h>
|
||||
#include <Interpreters/Access/InterpreterShowCreateAccessEntityQuery.h>
|
||||
#include <Interpreters/Access/InterpreterShowGrantsQuery.h>
|
||||
#include <Parsers/Access/ASTCreateQuotaQuery.h>
|
||||
#include <Parsers/Access/ASTCreateRoleQuery.h>
|
||||
#include <Parsers/Access/ASTCreateRowPolicyQuery.h>
|
||||
#include <Parsers/Access/ASTCreateSettingsProfileQuery.h>
|
||||
#include <Parsers/Access/ASTCreateUserQuery.h>
|
||||
#include <Parsers/Access/ASTGrantQuery.h>
|
||||
#include <Parsers/ParserAttachAccessEntity.h>
|
||||
#include <Parsers/formatAST.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
|
||||
@ -1,51 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#include <Parsers/Access/ASTCreateQuotaQuery.h>
|
||||
#include <Parsers/Access/ASTCreateRoleQuery.h>
|
||||
#include <Parsers/Access/ASTCreateRowPolicyQuery.h>
|
||||
#include <Parsers/Access/ASTCreateSettingsProfileQuery.h>
|
||||
#include <Parsers/Access/ASTCreateUserQuery.h>
|
||||
#include <Parsers/Access/ASTGrantQuery.h>
|
||||
#include <Parsers/Access/ParserCreateQuotaQuery.h>
|
||||
#include <Parsers/Access/ParserCreateRoleQuery.h>
|
||||
#include <Parsers/Access/ParserCreateRowPolicyQuery.h>
|
||||
#include <Parsers/Access/ParserCreateSettingsProfileQuery.h>
|
||||
#include <Parsers/Access/ParserCreateUserQuery.h>
|
||||
#include <Parsers/Access/ParserGrantQuery.h>
|
||||
#include <base/types.h>
|
||||
#include <memory>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Special parser for the 'ATTACH access entity' queries.
|
||||
class ParserAttachAccessEntity : public IParserBase
|
||||
{
|
||||
protected:
|
||||
const char * getName() const override { return "ATTACH access entity query"; }
|
||||
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
|
||||
{
|
||||
ParserCreateUserQuery create_user_p;
|
||||
ParserCreateRoleQuery create_role_p;
|
||||
ParserCreateRowPolicyQuery create_policy_p;
|
||||
ParserCreateQuotaQuery create_quota_p;
|
||||
ParserCreateSettingsProfileQuery create_profile_p;
|
||||
ParserGrantQuery grant_p;
|
||||
|
||||
create_user_p.useAttachMode();
|
||||
create_role_p.useAttachMode();
|
||||
create_policy_p.useAttachMode();
|
||||
create_quota_p.useAttachMode();
|
||||
create_profile_p.useAttachMode();
|
||||
grant_p.useAttachMode();
|
||||
|
||||
return create_user_p.parse(pos, node, expected) || create_role_p.parse(pos, node, expected)
|
||||
|| create_policy_p.parse(pos, node, expected) || create_quota_p.parse(pos, node, expected)
|
||||
|| create_profile_p.parse(pos, node, expected) || grant_p.parse(pos, node, expected);
|
||||
}
|
||||
};
|
||||
|
||||
struct IAccessEntity;
|
||||
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
|
||||
|
||||
|
||||
@ -514,7 +514,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
|
||||
else
|
||||
{
|
||||
/// These embedded files added during build with some cmake magic.
|
||||
/// Look at the end of programs/sever/CMakeLists.txt.
|
||||
/// Look at the end of programs/server/CMakeLists.txt.
|
||||
std::string embedded_name;
|
||||
if (path == "config.xml")
|
||||
embedded_name = "embedded.xml";
|
||||
|
||||
@ -619,6 +619,8 @@
|
||||
M(648, WRONG_DDL_RENAMING_SETTINGS) \
|
||||
M(649, INVALID_TRANSACTION) \
|
||||
M(650, SERIALIZATION_ERROR) \
|
||||
M(651, CAPN_PROTO_BAD_TYPE) \
|
||||
M(652, ONLY_NULLS_WHILE_READING_SCHEMA) \
|
||||
\
|
||||
M(999, KEEPER_EXCEPTION) \
|
||||
M(1000, POCO_EXCEPTION) \
|
||||
|
||||
@ -11,6 +11,13 @@ class NetException : public Exception
|
||||
public:
|
||||
NetException(const std::string & msg, int code) : Exception(msg, code) {}
|
||||
|
||||
// Format message with fmt::format, like the logging functions.
|
||||
template <typename... Args>
|
||||
NetException(int code, fmt::format_string<Args...> fmt, Args &&... args)
|
||||
: Exception(fmt::format(fmt, std::forward<Args>(args)...), code)
|
||||
{
|
||||
}
|
||||
|
||||
NetException * clone() const override { return new NetException(*this); }
|
||||
void rethrow() const override { throw *this; }
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ public:
|
||||
{
|
||||
}
|
||||
|
||||
size_t totalRanges() const { return static_cast<size_t>(round(static_cast<float>(total_size - from) / range_step)); }
|
||||
size_t totalRanges() const { return static_cast<size_t>(ceil(static_cast<float>(total_size - from) / range_step)); }
|
||||
|
||||
using Range = std::pair<size_t, size_t>;
|
||||
|
||||
|
||||
@ -265,24 +265,26 @@ void TaskStatsInfoGetter::getStat(::taskstats & out_stats, pid_t tid) const
|
||||
{
|
||||
NetlinkMessage answer = query(netlink_socket_fd, taskstats_family_id, tid, TASKSTATS_CMD_GET, TASKSTATS_CMD_ATTR_PID, &tid, sizeof(tid));
|
||||
|
||||
const NetlinkMessage::Attribute * attr = &answer.payload.attribute;
|
||||
if (attr->header.nla_type != TASKSTATS_TYPE_AGGR_PID)
|
||||
throw Exception("Expected TASKSTATS_TYPE_AGGR_PID", ErrorCodes::NETLINK_ERROR);
|
||||
for (const NetlinkMessage::Attribute * attr = &answer.payload.attribute;
|
||||
attr < answer.end();
|
||||
attr = attr->next())
|
||||
{
|
||||
if (attr->header.nla_type == TASKSTATS_TYPE_AGGR_TGID || attr->header.nla_type == TASKSTATS_TYPE_AGGR_PID)
|
||||
{
|
||||
for (const NetlinkMessage::Attribute * nested_attr = reinterpret_cast<const NetlinkMessage::Attribute *>(attr->payload);
|
||||
nested_attr < attr->next();
|
||||
nested_attr = nested_attr->next())
|
||||
{
|
||||
if (nested_attr->header.nla_type == TASKSTATS_TYPE_STATS)
|
||||
{
|
||||
out_stats = unalignedLoad<::taskstats>(nested_attr->payload);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// TASKSTATS_TYPE_AGGR_PID
|
||||
const NetlinkMessage::Attribute * nested_attr = reinterpret_cast<const NetlinkMessage::Attribute *>(attr->payload);
|
||||
if (nested_attr->header.nla_type != TASKSTATS_TYPE_PID)
|
||||
throw Exception("Expected TASKSTATS_TYPE_PID", ErrorCodes::NETLINK_ERROR);
|
||||
if (nested_attr == nested_attr->next())
|
||||
throw Exception("No TASKSTATS_TYPE_STATS packet after TASKSTATS_TYPE_PID", ErrorCodes::NETLINK_ERROR);
|
||||
nested_attr = nested_attr->next();
|
||||
if (nested_attr->header.nla_type != TASKSTATS_TYPE_STATS)
|
||||
throw Exception("Expected TASKSTATS_TYPE_STATS", ErrorCodes::NETLINK_ERROR);
|
||||
|
||||
out_stats = unalignedLoad<::taskstats>(nested_attr->payload);
|
||||
|
||||
if (attr->next() != answer.end())
|
||||
throw Exception("Unexpected end of response", ErrorCodes::NETLINK_ERROR);
|
||||
throw Exception("There is no TASKSTATS_TYPE_STATS attribute in the Netlink response", ErrorCodes::NETLINK_ERROR);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -67,20 +67,6 @@ namespace ProfileEvents
|
||||
namespace DB
|
||||
{
|
||||
|
||||
const char * TasksStatsCounters::metricsProviderString(MetricsProvider provider)
|
||||
{
|
||||
switch (provider)
|
||||
{
|
||||
case MetricsProvider::None:
|
||||
return "none";
|
||||
case MetricsProvider::Procfs:
|
||||
return "procfs";
|
||||
case MetricsProvider::Netlink:
|
||||
return "netlink";
|
||||
}
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
bool TasksStatsCounters::checkIfAvailable()
|
||||
{
|
||||
return findBestAvailableProvider() != MetricsProvider::None;
|
||||
|
||||
@ -176,17 +176,7 @@ extern PerfEventsCounters current_thread_counters;
|
||||
class TasksStatsCounters
|
||||
{
|
||||
public:
|
||||
enum class MetricsProvider
|
||||
{
|
||||
None,
|
||||
Procfs,
|
||||
Netlink,
|
||||
};
|
||||
|
||||
static const char * metricsProviderString(MetricsProvider provider);
|
||||
static bool checkIfAvailable();
|
||||
static MetricsProvider findBestAvailableProvider();
|
||||
|
||||
static std::unique_ptr<TasksStatsCounters> create(UInt64 tid);
|
||||
|
||||
void reset();
|
||||
@ -196,8 +186,16 @@ private:
|
||||
::taskstats stats; //-V730_NOINIT
|
||||
std::function<::taskstats()> stats_getter;
|
||||
|
||||
enum class MetricsProvider
|
||||
{
|
||||
None,
|
||||
Procfs,
|
||||
Netlink
|
||||
};
|
||||
|
||||
explicit TasksStatsCounters(UInt64 tid, MetricsProvider provider);
|
||||
|
||||
static MetricsProvider findBestAvailableProvider();
|
||||
static void incrementProfileEvents(const ::taskstats & prev, const ::taskstats & curr, ProfileEvents::Counters & profile_events);
|
||||
};
|
||||
|
||||
|
||||
22
src/Common/tests/gtest_range_generator.cpp
Normal file
22
src/Common/tests/gtest_range_generator.cpp
Normal file
@ -0,0 +1,22 @@
|
||||
#include <Common/RangeGenerator.h>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
using namespace DB;
|
||||
|
||||
|
||||
TEST(RangeGenerator, Common)
|
||||
{
|
||||
RangeGenerator g{25, 10};
|
||||
EXPECT_EQ(g.totalRanges(), 3);
|
||||
|
||||
std::vector<RangeGenerator::Range> ranges{{0, 10}, {10, 20}, {20, 25}};
|
||||
for (size_t i = 0; i < 3; ++i)
|
||||
{
|
||||
auto r = g.nextRange();
|
||||
EXPECT_TRUE(r);
|
||||
EXPECT_EQ(r, ranges[i]);
|
||||
}
|
||||
|
||||
auto r = g.nextRange();
|
||||
EXPECT_TRUE(!r);
|
||||
}
|
||||
@ -28,7 +28,7 @@ namespace ErrorCodes
|
||||
extern const int FILE_DOESNT_EXIST;
|
||||
extern const int UNKNOWN_EXCEPTION;
|
||||
extern const int INCORRECT_DATA;
|
||||
extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
|
||||
extern const int CAPN_PROTO_BAD_TYPE;
|
||||
}
|
||||
|
||||
capnp::StructSchema CapnProtoSchemaParser::getMessageSchema(const FormatSchemaInfo & schema_info)
|
||||
@ -447,7 +447,7 @@ static DataTypePtr getEnumDataTypeFromEnumSchema(const capnp::EnumSchema & enum_
|
||||
if (enumerants.size() < 32768)
|
||||
return getEnumDataTypeFromEnumerants<Int16>(enumerants);
|
||||
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "ClickHouse supports only 8 and 16-but Enums");
|
||||
throw Exception(ErrorCodes::CAPN_PROTO_BAD_TYPE, "ClickHouse supports only 8 and 16-bit Enums");
|
||||
}
|
||||
|
||||
static DataTypePtr getDataTypeFromCapnProtoType(const capnp::Type & capnp_type)
|
||||
@ -495,17 +495,17 @@ static DataTypePtr getDataTypeFromCapnProtoType(const capnp::Type & capnp_type)
|
||||
{
|
||||
auto fields = struct_schema.getUnionFields();
|
||||
if (fields.size() != 2 || (!fields[0].getType().isVoid() && !fields[1].getType().isVoid()))
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Unions are not supported");
|
||||
throw Exception(ErrorCodes::CAPN_PROTO_BAD_TYPE, "Unions are not supported");
|
||||
auto value_type = fields[0].getType().isVoid() ? fields[1].getType() : fields[0].getType();
|
||||
if (value_type.isStruct() || value_type.isList())
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Tuples and Lists cannot be inside Nullable");
|
||||
throw Exception(ErrorCodes::CAPN_PROTO_BAD_TYPE, "Tuples and Lists cannot be inside Nullable");
|
||||
|
||||
auto nested_type = getDataTypeFromCapnProtoType(value_type);
|
||||
return std::make_shared<DataTypeNullable>(nested_type);
|
||||
}
|
||||
|
||||
if (checkIfStructContainsUnnamedUnion(struct_schema))
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Unnamed union is not supported");
|
||||
throw Exception(ErrorCodes::CAPN_PROTO_BAD_TYPE, "Unnamed union is not supported");
|
||||
|
||||
/// Treat Struct as Tuple.
|
||||
DataTypes nested_types;
|
||||
@ -518,14 +518,14 @@ static DataTypePtr getDataTypeFromCapnProtoType(const capnp::Type & capnp_type)
|
||||
return std::make_shared<DataTypeTuple>(std::move(nested_types), std::move(nested_names));
|
||||
}
|
||||
default:
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Unsupported CapnProtoType: {}", getCapnProtoFullTypeName(capnp_type));
|
||||
throw Exception(ErrorCodes::CAPN_PROTO_BAD_TYPE, "Unsupported CapnProtoType: {}", getCapnProtoFullTypeName(capnp_type));
|
||||
}
|
||||
}
|
||||
|
||||
NamesAndTypesList capnProtoSchemaToCHSchema(const capnp::StructSchema & schema)
|
||||
{
|
||||
if (checkIfStructContainsUnnamedUnion(schema))
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Unnamed union is not supported");
|
||||
throw Exception(ErrorCodes::CAPN_PROTO_BAD_TYPE, "Unnamed union is not supported");
|
||||
|
||||
NamesAndTypesList names_and_types;
|
||||
for (auto field : schema.getNonUnionFields())
|
||||
|
||||
@ -374,7 +374,7 @@ String FormatFactory::getContentType(
|
||||
SchemaReaderPtr FormatFactory::getSchemaReader(
|
||||
const String & name,
|
||||
ReadBuffer & buf,
|
||||
ContextPtr context,
|
||||
ContextPtr & context,
|
||||
const std::optional<FormatSettings> & _format_settings) const
|
||||
{
|
||||
const auto & schema_reader_creator = dict.at(name).schema_reader_creator;
|
||||
@ -387,7 +387,7 @@ SchemaReaderPtr FormatFactory::getSchemaReader(
|
||||
|
||||
ExternalSchemaReaderPtr FormatFactory::getExternalSchemaReader(
|
||||
const String & name,
|
||||
ContextPtr context,
|
||||
ContextPtr & context,
|
||||
const std::optional<FormatSettings> & _format_settings) const
|
||||
{
|
||||
const auto & external_schema_reader_creator = dict.at(name).external_schema_reader_creator;
|
||||
|
||||
@ -160,12 +160,12 @@ public:
|
||||
SchemaReaderPtr getSchemaReader(
|
||||
const String & name,
|
||||
ReadBuffer & buf,
|
||||
ContextPtr context,
|
||||
ContextPtr & context,
|
||||
const std::optional<FormatSettings> & format_settings = std::nullopt) const;
|
||||
|
||||
ExternalSchemaReaderPtr getExternalSchemaReader(
|
||||
const String & name,
|
||||
ContextPtr context,
|
||||
ContextPtr & context,
|
||||
const std::optional<FormatSettings> & format_settings = std::nullopt) const;
|
||||
|
||||
void registerFileSegmentationEngine(const String & name, FileSegmentationEngine file_segmentation_engine);
|
||||
|
||||
@ -15,8 +15,10 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
|
||||
extern const int EMPTY_DATA_PASSED;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int ONLY_NULLS_WHILE_READING_SCHEMA;
|
||||
extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
|
||||
}
|
||||
|
||||
static std::optional<NamesAndTypesList> getOrderedColumnsList(
|
||||
@ -41,11 +43,17 @@ static std::optional<NamesAndTypesList> getOrderedColumnsList(
|
||||
return res;
|
||||
}
|
||||
|
||||
bool isRetryableSchemaInferenceError(int code)
|
||||
{
|
||||
return code == ErrorCodes::EMPTY_DATA_PASSED || code == ErrorCodes::ONLY_NULLS_WHILE_READING_SCHEMA;
|
||||
}
|
||||
|
||||
ColumnsDescription readSchemaFromFormat(
|
||||
const String & format_name,
|
||||
const std::optional<FormatSettings> & format_settings,
|
||||
ReadBufferCreator read_buffer_creator,
|
||||
ContextPtr context,
|
||||
ReadBufferIterator & read_buffer_iterator,
|
||||
bool retry,
|
||||
ContextPtr & context,
|
||||
std::unique_ptr<ReadBuffer> & buf_out)
|
||||
{
|
||||
NamesAndTypesList names_and_types;
|
||||
@ -63,20 +71,43 @@ ColumnsDescription readSchemaFromFormat(
|
||||
}
|
||||
else if (FormatFactory::instance().checkIfFormatHasSchemaReader(format_name))
|
||||
{
|
||||
buf_out = read_buffer_creator();
|
||||
if (buf_out->eof())
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file, file is empty", format_name);
|
||||
std::string exception_messages;
|
||||
SchemaReaderPtr schema_reader;
|
||||
std::unique_ptr<ReadBuffer> buf;
|
||||
while ((buf = read_buffer_iterator()))
|
||||
{
|
||||
if (buf->eof())
|
||||
{
|
||||
auto exception_message = fmt::format("Cannot extract table structure from {} format file, file is emptyg", format_name);
|
||||
|
||||
auto schema_reader = FormatFactory::instance().getSchemaReader(format_name, *buf_out, context, format_settings);
|
||||
try
|
||||
{
|
||||
names_and_types = schema_reader->readSchema();
|
||||
}
|
||||
catch (const DB::Exception & e)
|
||||
{
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file. Error: {}", format_name, e.message());
|
||||
if (!retry)
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, exception_message);
|
||||
|
||||
exception_messages += "\n" + exception_message;
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
schema_reader = FormatFactory::instance().getSchemaReader(format_name, *buf, context, format_settings);
|
||||
names_and_types = schema_reader->readSchema();
|
||||
buf_out = std::move(buf);
|
||||
break;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
auto exception_message = getCurrentExceptionMessage(false);
|
||||
|
||||
if (!retry || !isRetryableSchemaInferenceError(getCurrentExceptionCode()))
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file. Error: {}", format_name, exception_message);
|
||||
|
||||
exception_messages += "\n" + exception_message;
|
||||
}
|
||||
}
|
||||
|
||||
if (names_and_types.empty())
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from files failed. Errors:{}", exception_messages);
|
||||
|
||||
/// If we have "INSERT SELECT" query then try to order
|
||||
/// columns as they are ordered in table schema for formats
|
||||
/// without strict column order (like JSON and TSKV).
|
||||
@ -99,10 +130,10 @@ ColumnsDescription readSchemaFromFormat(
|
||||
return ColumnsDescription(names_and_types);
|
||||
}
|
||||
|
||||
ColumnsDescription readSchemaFromFormat(const String & format_name, const std::optional<FormatSettings> & format_settings, ReadBufferCreator read_buffer_creator, ContextPtr context)
|
||||
ColumnsDescription readSchemaFromFormat(const String & format_name, const std::optional<FormatSettings> & format_settings, ReadBufferIterator & read_buffer_iterator, bool retry, ContextPtr & context)
|
||||
{
|
||||
std::unique_ptr<ReadBuffer> buf_out;
|
||||
return readSchemaFromFormat(format_name, format_settings, read_buffer_creator, context, buf_out);
|
||||
return readSchemaFromFormat(format_name, format_settings, read_buffer_iterator, retry, context, buf_out);
|
||||
}
|
||||
|
||||
DataTypePtr makeNullableRecursivelyAndCheckForNothing(DataTypePtr type)
|
||||
|
||||
@ -6,27 +6,32 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
using ReadBufferIterator = std::function<std::unique_ptr<ReadBuffer>()>;
|
||||
|
||||
/// Try to determine the schema of the data in specifying format.
|
||||
/// For formats that have an external schema reader, it will
|
||||
/// use it and won't create a read buffer.
|
||||
/// For formats that have a schema reader from the data,
|
||||
/// read buffer will be created by the provided creator and
|
||||
/// the schema will be extracted from the data.
|
||||
/// If format doesn't have any schema reader or a schema reader
|
||||
/// couldn't determine the schema, an exception will be thrown.
|
||||
using ReadBufferCreator = std::function<std::unique_ptr<ReadBuffer>()>;
|
||||
/// read buffer will be created by the provided iterator and
|
||||
/// the schema will be extracted from the data. If schema reader
|
||||
/// couldn't determine the schema we will try the next read buffer
|
||||
/// from provided iterator if it makes sense. If format doesn't
|
||||
/// have any schema reader or we couldn't determine the schema,
|
||||
/// an exception will be thrown.
|
||||
ColumnsDescription readSchemaFromFormat(
|
||||
const String & format_name,
|
||||
const std::optional<FormatSettings> & format_settings,
|
||||
ReadBufferCreator read_buffer_creator,
|
||||
ContextPtr context);
|
||||
ReadBufferIterator & read_buffer_iterator,
|
||||
bool retry,
|
||||
ContextPtr & context);
|
||||
|
||||
/// If ReadBuffer is created, it will be written to buf_out.
|
||||
ColumnsDescription readSchemaFromFormat(
|
||||
const String & format_name,
|
||||
const std::optional<FormatSettings> & format_settings,
|
||||
ReadBufferCreator read_buffer_creator,
|
||||
ContextPtr context,
|
||||
ReadBufferIterator & read_buffer_iterator,
|
||||
bool retry,
|
||||
ContextPtr & context,
|
||||
std::unique_ptr<ReadBuffer> & buf_out);
|
||||
|
||||
/// Make type Nullable recursively:
|
||||
|
||||
@ -193,6 +193,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
|
||||
if (const auto * ptype = typeid_cast<const DataTypeDecimal<Decimal32> *>(&type)) return convertDecimalType(src, *ptype);
|
||||
if (const auto * ptype = typeid_cast<const DataTypeDecimal<Decimal64> *>(&type)) return convertDecimalType(src, *ptype);
|
||||
if (const auto * ptype = typeid_cast<const DataTypeDecimal<Decimal128> *>(&type)) return convertDecimalType(src, *ptype);
|
||||
if (const auto * ptype = typeid_cast<const DataTypeDecimal<Decimal256> *>(&type)) return convertDecimalType(src, *ptype);
|
||||
|
||||
if (which_type.isEnum() && (src.getType() == Field::Types::UInt64 || src.getType() == Field::Types::Int64))
|
||||
{
|
||||
|
||||
33
src/Parsers/ParserAttachAccessEntity.cpp
Normal file
33
src/Parsers/ParserAttachAccessEntity.cpp
Normal file
@ -0,0 +1,33 @@
|
||||
#include <Parsers/ParserAttachAccessEntity.h>
|
||||
#include <Parsers/Access/ParserCreateQuotaQuery.h>
|
||||
#include <Parsers/Access/ParserCreateRoleQuery.h>
|
||||
#include <Parsers/Access/ParserCreateRowPolicyQuery.h>
|
||||
#include <Parsers/Access/ParserCreateSettingsProfileQuery.h>
|
||||
#include <Parsers/Access/ParserCreateUserQuery.h>
|
||||
#include <Parsers/Access/ParserGrantQuery.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
bool ParserAttachAccessEntity::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
{
|
||||
ParserCreateUserQuery create_user_p;
|
||||
ParserCreateRoleQuery create_role_p;
|
||||
ParserCreateRowPolicyQuery create_policy_p;
|
||||
ParserCreateQuotaQuery create_quota_p;
|
||||
ParserCreateSettingsProfileQuery create_profile_p;
|
||||
ParserGrantQuery grant_p;
|
||||
|
||||
create_user_p.useAttachMode();
|
||||
create_role_p.useAttachMode();
|
||||
create_policy_p.useAttachMode();
|
||||
create_quota_p.useAttachMode();
|
||||
create_profile_p.useAttachMode();
|
||||
grant_p.useAttachMode();
|
||||
|
||||
return create_user_p.parse(pos, node, expected) || create_role_p.parse(pos, node, expected)
|
||||
|| create_policy_p.parse(pos, node, expected) || create_quota_p.parse(pos, node, expected)
|
||||
|| create_profile_p.parse(pos, node, expected) || grant_p.parse(pos, node, expected);
|
||||
}
|
||||
|
||||
}
|
||||
18
src/Parsers/ParserAttachAccessEntity.h
Normal file
18
src/Parsers/ParserAttachAccessEntity.h
Normal file
@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include <Parsers/IParserBase.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Special parser for the 'ATTACH access entity' queries.
|
||||
class ParserAttachAccessEntity : public IParserBase
|
||||
{
|
||||
protected:
|
||||
const char * getName() const override { return "ATTACH access entity query"; }
|
||||
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
@ -3,13 +3,15 @@
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/Access/ASTCreateUserQuery.h>
|
||||
#include <Parsers/Access/ParserCreateUserQuery.h>
|
||||
#include <Parsers/ParserAlterQuery.h>
|
||||
#include <Parsers/ParserCreateQuery.h>
|
||||
#include <Parsers/ParserOptimizeQuery.h>
|
||||
#include <Parsers/ParserQueryWithOutput.h>
|
||||
#include <Parsers/ParserAttachAccessEntity.h>
|
||||
#include <Parsers/formatAST.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include "Access/AccessEntityIO.h"
|
||||
#include <string_view>
|
||||
#include <regex>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
@ -34,12 +34,12 @@ ExecutingGraph::Edge & ExecutingGraph::addEdge(Edges & edges, Edge edge, const I
|
||||
{
|
||||
auto it = processors_map.find(to);
|
||||
if (it == processors_map.end())
|
||||
{
|
||||
String msg = "Processor " + to->getName() + " was found as " + (edge.backward ? "input" : "output")
|
||||
+ " for processor " + from->getName() + ", but not found in list of processors.";
|
||||
|
||||
throw Exception(msg, ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Processor {} was found as {} for processor {}, but not found in list of processors",
|
||||
to->getName(),
|
||||
edge.backward ? "input" : "output",
|
||||
from->getName());
|
||||
|
||||
edge.to = it->second;
|
||||
auto & added_edge = edges.emplace_back(std::move(edge));
|
||||
@ -128,8 +128,7 @@ bool ExecutingGraph::expandPipeline(std::stack<uint64_t> & stack, uint64_t pid)
|
||||
{
|
||||
auto * processor = processors[nodes.size()].get();
|
||||
if (processors_map.contains(processor))
|
||||
throw Exception("Processor " + processor->getName() + " was already added to pipeline.",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Processor {} was already added to pipeline", processor->getName());
|
||||
|
||||
processors_map[processor] = nodes.size();
|
||||
nodes.emplace_back(std::make_unique<Node>(processor, nodes.size()));
|
||||
|
||||
@ -28,6 +28,23 @@ void ExecutorTasks::rethrowFirstThreadException()
|
||||
executor_context->rethrowExceptionIfHas();
|
||||
}
|
||||
|
||||
void ExecutorTasks::tryWakeUpAnyOtherThreadWithTasks(ExecutionThreadContext & self, std::unique_lock<std::mutex> & lock)
|
||||
{
|
||||
if (!task_queue.empty() && !threads_queue.empty() && !finished)
|
||||
{
|
||||
size_t next_thread = self.thread_number + 1 == num_threads ? 0 : (self.thread_number + 1);
|
||||
auto thread_to_wake = task_queue.getAnyThreadWithTasks(next_thread);
|
||||
|
||||
if (threads_queue.has(thread_to_wake))
|
||||
threads_queue.pop(thread_to_wake);
|
||||
else
|
||||
thread_to_wake = threads_queue.popAny();
|
||||
|
||||
lock.unlock();
|
||||
executor_contexts[thread_to_wake]->wakeUp();
|
||||
}
|
||||
}
|
||||
|
||||
void ExecutorTasks::tryGetTask(ExecutionThreadContext & context)
|
||||
{
|
||||
{
|
||||
@ -43,20 +60,7 @@ void ExecutorTasks::tryGetTask(ExecutionThreadContext & context)
|
||||
|
||||
if (context.hasTask())
|
||||
{
|
||||
if (!task_queue.empty() && !threads_queue.empty())
|
||||
{
|
||||
size_t next_thread = context.thread_number + 1 == num_threads ? 0 : (context.thread_number + 1);
|
||||
auto thread_to_wake = task_queue.getAnyThreadWithTasks(next_thread);
|
||||
|
||||
if (threads_queue.has(thread_to_wake))
|
||||
threads_queue.pop(thread_to_wake);
|
||||
else
|
||||
thread_to_wake = threads_queue.popAny();
|
||||
|
||||
lock.unlock();
|
||||
executor_contexts[thread_to_wake]->wakeUp();
|
||||
}
|
||||
|
||||
tryWakeUpAnyOtherThreadWithTasks(context, lock);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -120,20 +124,7 @@ void ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThrea
|
||||
queue.pop();
|
||||
}
|
||||
|
||||
if (!threads_queue.empty() && !task_queue.empty() && !finished)
|
||||
{
|
||||
size_t next_thread = context.thread_number + 1 == num_threads ? 0 : (context.thread_number + 1);
|
||||
auto thread_to_wake = task_queue.getAnyThreadWithTasks(next_thread);
|
||||
|
||||
if (threads_queue.has(thread_to_wake))
|
||||
threads_queue.pop(thread_to_wake);
|
||||
else
|
||||
thread_to_wake = threads_queue.popAny();
|
||||
|
||||
lock.unlock();
|
||||
|
||||
executor_contexts[thread_to_wake]->wakeUp();
|
||||
}
|
||||
tryWakeUpAnyOtherThreadWithTasks(context, lock);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -50,6 +50,7 @@ public:
|
||||
|
||||
void rethrowFirstThreadException();
|
||||
|
||||
void tryWakeUpAnyOtherThreadWithTasks(ExecutionThreadContext & self, std::unique_lock<std::mutex> & lock);
|
||||
void tryGetTask(ExecutionThreadContext & context);
|
||||
void pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context);
|
||||
|
||||
|
||||
@ -185,10 +185,10 @@ void PipelineExecutor::executeSingleThread(size_t thread_num)
|
||||
auto & context = tasks.getThreadContext(thread_num);
|
||||
LOG_TRACE(log,
|
||||
"Thread finished. Total time: {} sec. Execution time: {} sec. Processing time: {} sec. Wait time: {} sec.",
|
||||
(context.total_time_ns / 1e9),
|
||||
(context.execution_time_ns / 1e9),
|
||||
(context.processing_time_ns / 1e9),
|
||||
(context.wait_time_ns / 1e9));
|
||||
context.total_time_ns / 1e9,
|
||||
context.execution_time_ns / 1e9,
|
||||
context.processing_time_ns / 1e9,
|
||||
context.wait_time_ns / 1e9);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -77,7 +77,7 @@ size_t StreamingFormatExecutor::execute()
|
||||
case IProcessor::Status::NeedData:
|
||||
case IProcessor::Status::Async:
|
||||
case IProcessor::Status::ExpandPipeline:
|
||||
throw Exception("Source processor returned status " + IProcessor::statusToName(status), ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Source processor returned status {}", IProcessor::statusToName(status));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -25,7 +25,7 @@ public:
|
||||
size_t getAnyThreadWithTasks(size_t from_thread = 0)
|
||||
{
|
||||
if (num_tasks == 0)
|
||||
throw Exception("TaskQueue is empty.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("TaskQueue is empty", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
for (size_t i = 0; i < queues.size(); ++i)
|
||||
{
|
||||
@ -37,7 +37,7 @@ public:
|
||||
from_thread = 0;
|
||||
}
|
||||
|
||||
throw Exception("TaskQueue is empty.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("TaskQueue is empty", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
Task * pop(size_t thread_num)
|
||||
|
||||
@ -36,7 +36,7 @@ struct ThreadsQueue
|
||||
void push(size_t thread)
|
||||
{
|
||||
if (unlikely(has(thread)))
|
||||
throw Exception("Can't push thread because it is already in threads queue.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Can't push thread because it is already in threads queue", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
swapThreads(thread, stack[stack_size]);
|
||||
++stack_size;
|
||||
@ -45,7 +45,7 @@ struct ThreadsQueue
|
||||
void pop(size_t thread)
|
||||
{
|
||||
if (unlikely(!has(thread)))
|
||||
throw Exception("Can't pop thread because it is not in threads queue.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Can't pop thread because it is not in threads queue", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
--stack_size;
|
||||
swapThreads(thread, stack[stack_size]);
|
||||
@ -54,7 +54,7 @@ struct ThreadsQueue
|
||||
size_t popAny()
|
||||
{
|
||||
if (unlikely(stack_size == 0))
|
||||
throw Exception("Can't pop from empty queue.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Can't pop from empty queue", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
--stack_size;
|
||||
return stack[stack_size];
|
||||
|
||||
@ -8,7 +8,10 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
|
||||
extern const int ONLY_NULLS_WHILE_READING_SCHEMA;
|
||||
extern const int TYPE_MISMATCH;
|
||||
extern const int INCORRECT_DATA;
|
||||
extern const int EMPTY_DATA_PASSED;
|
||||
}
|
||||
|
||||
static void chooseResultType(
|
||||
@ -36,7 +39,7 @@ static void chooseResultType(
|
||||
type = default_type;
|
||||
else
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
ErrorCodes::TYPE_MISMATCH,
|
||||
"Automatically defined type {} for column {} in row {} differs from type defined by previous rows: {}",
|
||||
type->getName(),
|
||||
column_name,
|
||||
@ -51,7 +54,7 @@ static void checkTypeAndAppend(NamesAndTypesList & result, DataTypePtr & type, c
|
||||
{
|
||||
if (!default_type)
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
ErrorCodes::ONLY_NULLS_WHILE_READING_SCHEMA,
|
||||
"Cannot determine table structure by first {} rows of data, because some columns contain only Nulls. To increase the maximum "
|
||||
"number of rows to read for structure determination, use setting input_format_max_rows_to_read_for_schema_inference",
|
||||
max_rows_to_read);
|
||||
@ -100,7 +103,7 @@ NamesAndTypesList IRowSchemaReader::readSchema()
|
||||
break;
|
||||
|
||||
if (new_data_types.size() != data_types.size())
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Rows have different amount of values");
|
||||
throw Exception(ErrorCodes::INCORRECT_DATA, "Rows have different amount of values");
|
||||
|
||||
for (size_t i = 0; i != data_types.size(); ++i)
|
||||
{
|
||||
@ -114,7 +117,7 @@ NamesAndTypesList IRowSchemaReader::readSchema()
|
||||
|
||||
/// Check that we read at list one column.
|
||||
if (data_types.empty())
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot read rows from the data");
|
||||
throw Exception(ErrorCodes::EMPTY_DATA_PASSED, "Cannot read rows from the data");
|
||||
|
||||
/// If column names weren't set, use default names 'c1', 'c2', ...
|
||||
if (column_names.empty())
|
||||
@ -126,7 +129,7 @@ NamesAndTypesList IRowSchemaReader::readSchema()
|
||||
/// If column names were set, check that the number of names match the number of types.
|
||||
else if (column_names.size() != data_types.size())
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
ErrorCodes::INCORRECT_DATA,
|
||||
"The number of column names {} differs with the number of types {}", column_names.size(), data_types.size());
|
||||
|
||||
NamesAndTypesList result;
|
||||
@ -192,7 +195,7 @@ NamesAndTypesList IRowWithNamesSchemaReader::readSchema()
|
||||
|
||||
/// Check that we read at list one column.
|
||||
if (names_to_types.empty())
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot read rows from the data");
|
||||
throw Exception(ErrorCodes::EMPTY_DATA_PASSED, "Cannot read rows from the data");
|
||||
|
||||
NamesAndTypesList result;
|
||||
for (auto & name : names_order)
|
||||
|
||||
@ -11,7 +11,6 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int INCORRECT_DATA;
|
||||
extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
|
||||
}
|
||||
|
||||
RowInputFormatWithNamesAndTypes::RowInputFormatWithNamesAndTypes(
|
||||
@ -319,7 +318,7 @@ NamesAndTypesList FormatWithNamesAndTypesSchemaReader::readSchema()
|
||||
std::vector<String> data_type_names = format_reader->readTypes();
|
||||
if (data_type_names.size() != names.size())
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
ErrorCodes::INCORRECT_DATA,
|
||||
"The number of column names {} differs with the number of types {}", names.size(), data_type_names.size());
|
||||
|
||||
NamesAndTypesList result;
|
||||
|
||||
@ -67,8 +67,7 @@ void IInflatingTransform::work()
|
||||
if (can_generate)
|
||||
{
|
||||
if (generated)
|
||||
throw Exception("IInflatingTransform cannot consume chunk because it already was generated",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("IInflatingTransform cannot consume chunk because it already was generated", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
current_chunk = generate();
|
||||
generated = true;
|
||||
@ -77,8 +76,7 @@ void IInflatingTransform::work()
|
||||
else
|
||||
{
|
||||
if (!has_input)
|
||||
throw Exception("IInflatingTransform cannot consume chunk because it wasn't read",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("IInflatingTransform cannot consume chunk because it wasn't read", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
consume(std::move(current_chunk));
|
||||
has_input = false;
|
||||
|
||||
@ -178,7 +178,7 @@ public:
|
||||
*/
|
||||
virtual Status prepare()
|
||||
{
|
||||
throw Exception("Method 'prepare' is not implemented for " + getName() + " processor", ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'prepare' is not implemented for {} processor", getName());
|
||||
}
|
||||
|
||||
using PortNumbers = std::vector<UInt64>;
|
||||
@ -193,7 +193,7 @@ public:
|
||||
*/
|
||||
virtual void work()
|
||||
{
|
||||
throw Exception("Method 'work' is not implemented for " + getName() + " processor", ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'work' is not implemented for {} processor", getName());
|
||||
}
|
||||
|
||||
/** Executor must call this method when 'prepare' returned Async.
|
||||
@ -212,7 +212,7 @@ public:
|
||||
*/
|
||||
virtual int schedule()
|
||||
{
|
||||
throw Exception("Method 'schedule' is not implemented for " + getName() + " processor", ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'schedule' is not implemented for {} processor", getName());
|
||||
}
|
||||
|
||||
/** You must call this method if 'prepare' returned ExpandPipeline.
|
||||
@ -226,7 +226,7 @@ public:
|
||||
*/
|
||||
virtual Processors expandPipeline()
|
||||
{
|
||||
throw Exception("Method 'expandPipeline' is not implemented for " + getName() + " processor", ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'expandPipeline' is not implemented for {} processor", getName());
|
||||
}
|
||||
|
||||
/// In case if query was cancelled executor will wait till all processors finish their jobs.
|
||||
@ -258,7 +258,7 @@ public:
|
||||
++number;
|
||||
}
|
||||
|
||||
throw Exception("Can't find input port for " + getName() + " processor", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't find input port for {} processor", getName());
|
||||
}
|
||||
|
||||
UInt64 getOutputPortNumber(const OutputPort * output_port) const
|
||||
@ -272,7 +272,7 @@ public:
|
||||
++number;
|
||||
}
|
||||
|
||||
throw Exception("Can't find output port for " + getName() + " processor", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't find output port for {} processor", getName());
|
||||
}
|
||||
|
||||
const auto & getInputs() const { return inputs; }
|
||||
|
||||
@ -46,7 +46,7 @@ public:
|
||||
|
||||
virtual void transform(Chunk &)
|
||||
{
|
||||
throw Exception("Method transform is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method transform is not implemented for {}", getName());
|
||||
}
|
||||
|
||||
Status prepare() override;
|
||||
|
||||
@ -19,7 +19,7 @@ LimitTransform::LimitTransform(
|
||||
, with_ties(with_ties_), description(std::move(description_))
|
||||
{
|
||||
if (num_streams != 1 && with_ties)
|
||||
throw Exception("Cannot use LimitTransform with multiple ports and ties.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot use LimitTransform with multiple ports and ties", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
ports_data.resize(num_streams);
|
||||
|
||||
@ -86,8 +86,7 @@ IProcessor::Status LimitTransform::prepare(
|
||||
return;
|
||||
default:
|
||||
throw Exception(
|
||||
"Unexpected status for LimitTransform::preparePair : " + IProcessor::statusToName(status),
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
ErrorCodes::LOGICAL_ERROR, "Unexpected status for LimitTransform::preparePair : {}", IProcessor::statusToName(status));
|
||||
}
|
||||
};
|
||||
|
||||
@ -126,7 +125,7 @@ IProcessor::Status LimitTransform::prepare(
|
||||
LimitTransform::Status LimitTransform::prepare()
|
||||
{
|
||||
if (ports_data.size() != 1)
|
||||
throw Exception("prepare without arguments is not supported for multi-port LimitTransform.",
|
||||
throw Exception("prepare without arguments is not supported for multi-port LimitTransform",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
return prepare({0}, {0});
|
||||
|
||||
@ -153,7 +153,7 @@ struct RowRef
|
||||
return true;
|
||||
}
|
||||
|
||||
bool hasEqualSortColumnsWith(const RowRef & other)
|
||||
bool hasEqualSortColumnsWith(const RowRef & other) const
|
||||
{
|
||||
return checkEquals(num_columns, sort_columns, row_num, other.sort_columns, other.row_num);
|
||||
}
|
||||
@ -197,7 +197,7 @@ struct RowRefWithOwnedChunk
|
||||
sort_columns = &owned_chunk->sort_columns;
|
||||
}
|
||||
|
||||
bool hasEqualSortColumnsWith(const RowRefWithOwnedChunk & other)
|
||||
bool hasEqualSortColumnsWith(const RowRefWithOwnedChunk & other) const
|
||||
{
|
||||
return RowRef::checkEquals(sort_columns->size(), sort_columns->data(), row_num,
|
||||
other.sort_columns->data(), other.row_num);
|
||||
|
||||
@ -32,9 +32,7 @@ OffsetTransform::OffsetTransform(
|
||||
}
|
||||
|
||||
|
||||
IProcessor::Status OffsetTransform::prepare(
|
||||
const PortNumbers & updated_input_ports,
|
||||
const PortNumbers & updated_output_ports)
|
||||
IProcessor::Status OffsetTransform::prepare(const PortNumbers & updated_input_ports, const PortNumbers & updated_output_ports)
|
||||
{
|
||||
bool has_full_port = false;
|
||||
|
||||
@ -63,9 +61,7 @@ IProcessor::Status OffsetTransform::prepare(
|
||||
return;
|
||||
default:
|
||||
throw Exception(
|
||||
"Unexpected status for OffsetTransform::preparePair : " + IProcessor::statusToName(status),
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
ErrorCodes::LOGICAL_ERROR, "Unexpected status for OffsetTransform::preparePair : {}", IProcessor::statusToName(status));
|
||||
}
|
||||
};
|
||||
|
||||
@ -88,7 +84,7 @@ IProcessor::Status OffsetTransform::prepare(
|
||||
OffsetTransform::Status OffsetTransform::prepare()
|
||||
{
|
||||
if (ports_data.size() != 1)
|
||||
throw Exception("prepare without arguments is not supported for multi-port OffsetTransform.",
|
||||
throw Exception("prepare without arguments is not supported for multi-port OffsetTransform",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
return prepare({0}, {0});
|
||||
|
||||
@ -16,7 +16,7 @@ void connect(OutputPort & output, InputPort & input)
|
||||
auto out_name = output.getProcessor().getName();
|
||||
auto in_name = input.getProcessor().getName();
|
||||
|
||||
assertCompatibleHeader(output.getHeader(), input.getHeader(), " function connect between " + out_name + " and " + in_name);
|
||||
assertCompatibleHeader(output.getHeader(), input.getHeader(), fmt::format(" function connect between {} and {}", out_name, in_name));
|
||||
|
||||
input.output_port = &output;
|
||||
output.input_port = &input;
|
||||
|
||||
@ -89,7 +89,7 @@ protected:
|
||||
DataPtr() : data(new Data())
|
||||
{
|
||||
if (unlikely((getUInt(data) & FLAGS_MASK) != 0))
|
||||
throw Exception("Not alignment memory for Port.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Not alignment memory for Port", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
/// Pointer can store flags in case of exception in swap.
|
||||
~DataPtr() { delete getPtr(getUInt(data) & PTR_MASK); }
|
||||
@ -133,7 +133,7 @@ protected:
|
||||
State() : data(new Data())
|
||||
{
|
||||
if (unlikely((getUInt(data) & FLAGS_MASK) != 0))
|
||||
throw Exception("Not alignment memory for Port.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Not alignment memory for Port", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
~State()
|
||||
@ -160,7 +160,7 @@ protected:
|
||||
/// throw Exception("Cannot push block to port which is not needed.", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (unlikely(flags & HAS_DATA))
|
||||
throw Exception("Cannot push block to port which already has data.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot push block to port which already has data", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
void ALWAYS_INLINE pull(DataPtr & data_, std::uintptr_t & flags, bool set_not_needed = false)
|
||||
@ -174,10 +174,10 @@ protected:
|
||||
|
||||
/// It's ok to check because this flag can be changed only by pulling thread.
|
||||
if (unlikely((flags & IS_NEEDED) == 0) && !set_not_needed)
|
||||
throw Exception("Cannot pull block from port which is not needed.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot pull block from port which is not needed", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (unlikely((flags & HAS_DATA) == 0))
|
||||
throw Exception("Cannot pull block from port which has no data.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot pull block from port which has no data", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
std::uintptr_t ALWAYS_INLINE setFlags(std::uintptr_t flags, std::uintptr_t mask)
|
||||
@ -289,13 +289,15 @@ public:
|
||||
{
|
||||
auto & chunk = data->chunk;
|
||||
|
||||
String msg = "Invalid number of columns in chunk pulled from OutputPort. Expected "
|
||||
+ std::to_string(header.columns()) + ", found " + std::to_string(chunk.getNumColumns()) + '\n';
|
||||
|
||||
msg += "Header: " + header.dumpStructure() + '\n';
|
||||
msg += "Chunk: " + chunk.dumpStructure() + '\n';
|
||||
|
||||
throw Exception(msg, ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Invalid number of columns in chunk pulled from OutputPort. Expected {}, found {}\n"
|
||||
"Header: {}\n"
|
||||
"Chunk: {}\n",
|
||||
header.columns(),
|
||||
chunk.getNumColumns(),
|
||||
header.dumpStructure(),
|
||||
chunk.dumpStructure());
|
||||
}
|
||||
|
||||
return std::move(*data);
|
||||
@ -403,14 +405,15 @@ public:
|
||||
{
|
||||
if (unlikely(!data_.exception && data_.chunk.getNumColumns() != header.columns()))
|
||||
{
|
||||
String msg = "Invalid number of columns in chunk pushed to OutputPort. Expected "
|
||||
+ std::to_string(header.columns())
|
||||
+ ", found " + std::to_string(data_.chunk.getNumColumns()) + '\n';
|
||||
|
||||
msg += "Header: " + header.dumpStructure() + '\n';
|
||||
msg += "Chunk: " + data_.chunk.dumpStructure() + '\n';
|
||||
|
||||
throw Exception(msg, ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Invalid number of columns in chunk pushed to OutputPort. Expected {}, found {}\n"
|
||||
"Header: {}\n"
|
||||
"Chunk: {}\n",
|
||||
header.columns(),
|
||||
data_.chunk.getNumColumns(),
|
||||
header.dumpStructure(),
|
||||
data_.chunk.dumpStructure());
|
||||
}
|
||||
|
||||
updateVersion();
|
||||
|
||||
@ -64,22 +64,25 @@ void QueryPlan::unitePlans(QueryPlanStepPtr step, std::vector<std::unique_ptr<Qu
|
||||
const auto & inputs = step->getInputStreams();
|
||||
size_t num_inputs = step->getInputStreams().size();
|
||||
if (num_inputs != plans.size())
|
||||
{
|
||||
throw Exception("Cannot unite QueryPlans using " + step->getName() +
|
||||
" because step has different number of inputs. "
|
||||
"Has " + std::to_string(plans.size()) + " plans "
|
||||
"and " + std::to_string(num_inputs) + " inputs", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot unite QueryPlans using {} because step has different number of inputs. Has {} plans and {} inputs",
|
||||
step->getName(),
|
||||
plans.size(),
|
||||
num_inputs);
|
||||
|
||||
for (size_t i = 0; i < num_inputs; ++i)
|
||||
{
|
||||
const auto & step_header = inputs[i].header;
|
||||
const auto & plan_header = plans[i]->getCurrentDataStream().header;
|
||||
if (!blocksHaveEqualStructure(step_header, plan_header))
|
||||
throw Exception("Cannot unite QueryPlans using " + step->getName() + " because "
|
||||
"it has incompatible header with plan " + root->step->getName() + " "
|
||||
"plan header: " + plan_header.dumpStructure() +
|
||||
"step header: " + step_header.dumpStructure(), ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot unite QueryPlans using {} because it has incompatible header with plan {} plan header: {} step header: {}",
|
||||
step->getName(),
|
||||
root->step->getName(),
|
||||
plan_header.dumpStructure(),
|
||||
step_header.dumpStructure());
|
||||
}
|
||||
|
||||
for (auto & plan : plans)
|
||||
@ -108,8 +111,10 @@ void QueryPlan::addStep(QueryPlanStepPtr step)
|
||||
if (num_input_streams == 0)
|
||||
{
|
||||
if (isInitialized())
|
||||
throw Exception("Cannot add step " + step->getName() + " to QueryPlan because "
|
||||
"step has no inputs, but QueryPlan is already initialized", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add step {} to QueryPlan because step has no inputs, but QueryPlan is already initialized",
|
||||
step->getName());
|
||||
|
||||
nodes.emplace_back(Node{.step = std::move(step)});
|
||||
root = &nodes.back();
|
||||
@ -119,25 +124,33 @@ void QueryPlan::addStep(QueryPlanStepPtr step)
|
||||
if (num_input_streams == 1)
|
||||
{
|
||||
if (!isInitialized())
|
||||
throw Exception("Cannot add step " + step->getName() + " to QueryPlan because "
|
||||
"step has input, but QueryPlan is not initialized", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add step {} to QueryPlan because step has input, but QueryPlan is not initialized",
|
||||
step->getName());
|
||||
|
||||
const auto & root_header = root->step->getOutputStream().header;
|
||||
const auto & step_header = step->getInputStreams().front().header;
|
||||
if (!blocksHaveEqualStructure(root_header, step_header))
|
||||
throw Exception("Cannot add step " + step->getName() + " to QueryPlan because "
|
||||
"it has incompatible header with root step " + root->step->getName() + " "
|
||||
"root header: " + root_header.dumpStructure() +
|
||||
"step header: " + step_header.dumpStructure(), ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add step {} to QueryPlan because it has incompatible header with root step {} root header: {} step header: {}",
|
||||
step->getName(),
|
||||
root->step->getName(),
|
||||
root_header.dumpStructure(),
|
||||
step_header.dumpStructure());
|
||||
|
||||
nodes.emplace_back(Node{.step = std::move(step), .children = {root}});
|
||||
root = &nodes.back();
|
||||
return;
|
||||
}
|
||||
|
||||
throw Exception("Cannot add step " + step->getName() + " to QueryPlan because it has " +
|
||||
std::to_string(num_input_streams) + " inputs but " + std::to_string(isInitialized() ? 1 : 0) +
|
||||
" input expected", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add step {} to QueryPlan because it has {} inputs but {} input expected",
|
||||
step->getName(),
|
||||
num_input_streams,
|
||||
isInitialized() ? 1 : 0);
|
||||
}
|
||||
|
||||
QueryPipelineBuilderPtr QueryPlan::buildQueryPipeline(
|
||||
|
||||
@ -343,12 +343,12 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in
|
||||
inputs_with_data.pop();
|
||||
|
||||
if (input_with_data.waiting_output == -1)
|
||||
throw Exception("No associated output for input with data.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("No associated output for input with data", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
auto & waiting_output = output_ports[input_with_data.waiting_output];
|
||||
|
||||
if (waiting_output.status == OutputStatus::NotActive)
|
||||
throw Exception("Invalid status NotActive for associated output.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Invalid status NotActive for associated output", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (waiting_output.status != OutputStatus::Finished)
|
||||
{
|
||||
|
||||
@ -337,8 +337,11 @@ void MySQLSource::initPositionMappingFromQueryResultStructure()
|
||||
if (!settings->fetch_by_name)
|
||||
{
|
||||
if (description.sample_block.columns() != connection->result.getNumFields())
|
||||
throw Exception{"mysqlxx::UseQueryResult contains " + toString(connection->result.getNumFields()) + " columns while "
|
||||
+ toString(description.sample_block.columns()) + " expected", ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH};
|
||||
throw Exception(
|
||||
ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH,
|
||||
"mysqlxx::UseQueryResult contains {} columns while {} expected",
|
||||
connection->result.getNumFields(),
|
||||
description.sample_block.columns());
|
||||
|
||||
for (const auto idx : collections::range(0, connection->result.getNumFields()))
|
||||
position_mapping[idx] = idx;
|
||||
@ -362,18 +365,10 @@ void MySQLSource::initPositionMappingFromQueryResultStructure()
|
||||
}
|
||||
|
||||
if (!missing_names.empty())
|
||||
{
|
||||
WriteBufferFromOwnString exception_message;
|
||||
for (auto iter = missing_names.begin(); iter != missing_names.end(); ++iter)
|
||||
{
|
||||
if (iter != missing_names.begin())
|
||||
exception_message << ", ";
|
||||
exception_message << *iter;
|
||||
}
|
||||
|
||||
throw Exception("mysqlxx::UseQueryResult must be contain the" + exception_message.str() + " columns.",
|
||||
ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH);
|
||||
}
|
||||
throw Exception(
|
||||
ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH,
|
||||
"mysqlxx::UseQueryResult must contain columns: {}",
|
||||
fmt::join(missing_names, ", "));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -18,10 +18,7 @@ class SQLiteSource : public SourceWithProgress
|
||||
using SQLitePtr = std::shared_ptr<sqlite3>;
|
||||
|
||||
public:
|
||||
SQLiteSource(SQLitePtr sqlite_db_,
|
||||
const String & query_str_,
|
||||
const Block & sample_block,
|
||||
UInt64 max_block_size_);
|
||||
SQLiteSource(SQLitePtr sqlite_db_, const String & query_str_, const Block & sample_block, UInt64 max_block_size_);
|
||||
|
||||
String getName() const override { return "SQLite"; }
|
||||
|
||||
|
||||
@ -125,7 +125,7 @@ public:
|
||||
ssize_t res = ::read(fd, internal_buffer.begin(), internal_buffer.size());
|
||||
|
||||
if (-1 == res && errno != EINTR)
|
||||
throwFromErrno("Cannot read from pipe ", ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR);
|
||||
throwFromErrno("Cannot read from pipe", ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR);
|
||||
|
||||
if (res == 0)
|
||||
break;
|
||||
@ -187,7 +187,7 @@ public:
|
||||
ssize_t res = ::write(fd, working_buffer.begin() + bytes_written, offset() - bytes_written);
|
||||
|
||||
if ((-1 == res || 0 == res) && errno != EINTR)
|
||||
throwFromErrno("Cannot write into pipe ", ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR);
|
||||
throwFromErrno("Cannot write into pipe", ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR);
|
||||
|
||||
if (res > 0)
|
||||
bytes_written += res;
|
||||
|
||||
@ -12,23 +12,27 @@ namespace ErrorCodes
|
||||
static void checkSingleInput(const IProcessor & transform)
|
||||
{
|
||||
if (transform.getInputs().size() != 1)
|
||||
throw Exception("Transform for chain should have single input, "
|
||||
"but " + transform.getName() + " has " +
|
||||
toString(transform.getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Transform for chain should have single input, but {} has {} inputs",
|
||||
transform.getName(),
|
||||
transform.getInputs().size());
|
||||
|
||||
if (transform.getInputs().front().isConnected())
|
||||
throw Exception("Transform for chain has connected input.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Transform for chain has connected input", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
static void checkSingleOutput(const IProcessor & transform)
|
||||
{
|
||||
if (transform.getOutputs().size() != 1)
|
||||
throw Exception("Transform for chain should have single output, "
|
||||
"but " + transform.getName() + " has " +
|
||||
toString(transform.getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Transform for chain should have single output, but {} has {} outputs",
|
||||
transform.getName(),
|
||||
transform.getOutputs().size());
|
||||
|
||||
if (transform.getOutputs().front().isConnected())
|
||||
throw Exception("Transform for chain has connected input.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Transform for chain has connected output", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
static void checkTransform(const IProcessor & transform)
|
||||
@ -40,7 +44,7 @@ static void checkTransform(const IProcessor & transform)
|
||||
static void checkInitialized(const std::list<ProcessorPtr> & processors)
|
||||
{
|
||||
if (processors.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Drain is not initialized");
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Chain is not initialized");
|
||||
}
|
||||
|
||||
Chain::Chain(ProcessorPtr processor)
|
||||
@ -61,15 +65,17 @@ Chain::Chain(std::list<ProcessorPtr> processors_) : processors(std::move(process
|
||||
{
|
||||
for (const auto & input : processor->getInputs())
|
||||
if (&input != &getInputPort() && !input.isConnected())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot initialize chain because there is a not connected input for {}",
|
||||
processor->getName());
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot initialize chain because there is a disconnected input for {}",
|
||||
processor->getName());
|
||||
|
||||
for (const auto & output : processor->getOutputs())
|
||||
if (&output != &getOutputPort() && !output.isConnected())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot initialize chain because there is a not connected output for {}",
|
||||
processor->getName());
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot initialize chain because there is a disconnected output for {}",
|
||||
processor->getName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Core/BackgroundSchedulePool.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/NetException.h>
|
||||
#include "Core/Protocol.h"
|
||||
#include <base/logger_useful.h>
|
||||
|
||||
@ -17,7 +18,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int UNKNOWN_PACKET_FROM_SERVER;
|
||||
extern const int UNEXPECTED_PACKET_FROM_SERVER;
|
||||
}
|
||||
|
||||
std::unique_ptr<ConnectionCollector> ConnectionCollector::connection_collector;
|
||||
@ -33,7 +34,7 @@ ConnectionCollector & ConnectionCollector::init(ContextMutablePtr global_context
|
||||
{
|
||||
if (connection_collector)
|
||||
{
|
||||
throw Exception("Connection collector is initialized twice. This is a bug.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Connection collector is initialized twice. This is a bug", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
connection_collector.reset(new ConnectionCollector(global_context_, max_threads));
|
||||
@ -90,13 +91,13 @@ void ConnectionCollector::drainConnections(IConnections & connections, bool thro
|
||||
break;
|
||||
|
||||
default:
|
||||
/// Connection should be closed in case of unknown packet,
|
||||
/// Connection should be closed in case of unexpected packet,
|
||||
/// since this means that the connection in some bad state.
|
||||
is_drained = false;
|
||||
throw Exception(
|
||||
ErrorCodes::UNKNOWN_PACKET_FROM_SERVER,
|
||||
"Unknown packet {} from one of the following replicas: {}",
|
||||
toString(packet.type),
|
||||
throw NetException(
|
||||
ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER,
|
||||
"Unexpected packet {} from one of the following replicas: {}. (expected EndOfStream, Log, ProfileEvents or Exception)",
|
||||
Protocol::Server::toString(packet.type),
|
||||
connections.dumpAddresses());
|
||||
}
|
||||
}
|
||||
|
||||
@ -61,15 +61,19 @@ void ExecutionSpeedLimits::throttle(
|
||||
{
|
||||
auto rows_per_second = read_rows / elapsed_seconds;
|
||||
if (min_execution_rps && rows_per_second < min_execution_rps)
|
||||
throw Exception("Query is executing too slow: " + toString(read_rows / elapsed_seconds)
|
||||
+ " rows/sec., minimum: " + toString(min_execution_rps),
|
||||
ErrorCodes::TOO_SLOW);
|
||||
throw Exception(
|
||||
ErrorCodes::TOO_SLOW,
|
||||
"Query is executing too slow: {} rows/sec., minimum: {}",
|
||||
read_rows / elapsed_seconds,
|
||||
min_execution_rps);
|
||||
|
||||
auto bytes_per_second = read_bytes / elapsed_seconds;
|
||||
if (min_execution_bps && bytes_per_second < min_execution_bps)
|
||||
throw Exception("Query is executing too slow: " + toString(read_bytes / elapsed_seconds)
|
||||
+ " bytes/sec., minimum: " + toString(min_execution_bps),
|
||||
ErrorCodes::TOO_SLOW);
|
||||
throw Exception(
|
||||
ErrorCodes::TOO_SLOW,
|
||||
"Query is executing too slow: {} bytes/sec., minimum: {}",
|
||||
read_bytes / elapsed_seconds,
|
||||
min_execution_bps);
|
||||
|
||||
/// If the predicted execution time is longer than `max_execution_time`.
|
||||
if (max_execution_time != 0 && total_rows_to_read && read_rows)
|
||||
@ -77,10 +81,12 @@ void ExecutionSpeedLimits::throttle(
|
||||
double estimated_execution_time_seconds = elapsed_seconds * (static_cast<double>(total_rows_to_read) / read_rows);
|
||||
|
||||
if (estimated_execution_time_seconds > max_execution_time.totalSeconds())
|
||||
throw Exception("Estimated query execution time (" + toString(estimated_execution_time_seconds) + " seconds)"
|
||||
+ " is too long. Maximum: " + toString(max_execution_time.totalSeconds())
|
||||
+ ". Estimated rows to process: " + toString(total_rows_to_read),
|
||||
ErrorCodes::TOO_SLOW);
|
||||
throw Exception(
|
||||
ErrorCodes::TOO_SLOW,
|
||||
"Estimated query execution time ({} seconds) is too long. Maximum: {}. Estimated rows to process: {}",
|
||||
estimated_execution_time_seconds,
|
||||
max_execution_time.totalSeconds(),
|
||||
total_rows_to_read);
|
||||
}
|
||||
|
||||
if (max_execution_rps && rows_per_second >= max_execution_rps)
|
||||
@ -92,12 +98,13 @@ void ExecutionSpeedLimits::throttle(
|
||||
}
|
||||
}
|
||||
|
||||
static bool handleOverflowMode(OverflowMode mode, const String & message, int code)
|
||||
template <typename... Args>
|
||||
static bool handleOverflowMode(OverflowMode mode, int code, fmt::format_string<Args...> fmt, Args &&... args)
|
||||
{
|
||||
switch (mode)
|
||||
{
|
||||
case OverflowMode::THROW:
|
||||
throw Exception(message, code);
|
||||
throw Exception(code, std::move(fmt), std::forward<Args>(args)...);
|
||||
case OverflowMode::BREAK:
|
||||
return false;
|
||||
default:
|
||||
@ -112,10 +119,12 @@ bool ExecutionSpeedLimits::checkTimeLimit(const Stopwatch & stopwatch, OverflowM
|
||||
auto elapsed_ns = stopwatch.elapsed();
|
||||
|
||||
if (elapsed_ns > static_cast<UInt64>(max_execution_time.totalMicroseconds()) * 1000)
|
||||
return handleOverflowMode(overflow_mode,
|
||||
"Timeout exceeded: elapsed " + toString(static_cast<double>(elapsed_ns) / 1000000000ULL)
|
||||
+ " seconds, maximum: " + toString(max_execution_time.totalMicroseconds() / 1000000.0),
|
||||
ErrorCodes::TIMEOUT_EXCEEDED);
|
||||
return handleOverflowMode(
|
||||
overflow_mode,
|
||||
ErrorCodes::TIMEOUT_EXCEEDED,
|
||||
"Timeout exceeded: elapsed {} seconds, maximum: {}",
|
||||
static_cast<double>(elapsed_ns) / 1000000000ULL,
|
||||
max_execution_time.totalMicroseconds() / 1000000.0);
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
@ -23,16 +23,18 @@ namespace ErrorCodes
|
||||
static void checkSource(const IProcessor & source)
|
||||
{
|
||||
if (!source.getInputs().empty())
|
||||
throw Exception("Source for pipe shouldn't have any input, but " + source.getName() + " has " +
|
||||
toString(source.getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Source for pipe shouldn't have any input, but {} has {} inputs",
|
||||
source.getName(),
|
||||
source.getInputs().size());
|
||||
|
||||
if (source.getOutputs().empty())
|
||||
throw Exception("Source for pipe should have single output, but it doesn't have any",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (source.getOutputs().size() > 1)
|
||||
throw Exception("Source for pipe should have single output, but " + source.getName() + " has " +
|
||||
toString(source.getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
if (source.getOutputs().size() != 1)
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Source for pipe should have single output, but {} has {} outputs",
|
||||
source.getName(),
|
||||
source.getOutputs().size());
|
||||
}
|
||||
|
||||
static OutputPort * uniteExtremes(const OutputPortRawPtrs & ports, const Block & header, Processors & processors)
|
||||
@ -112,8 +114,11 @@ PipelineResourcesHolder Pipe::detachResources()
|
||||
Pipe::Pipe(ProcessorPtr source, OutputPort * output, OutputPort * totals, OutputPort * extremes)
|
||||
{
|
||||
if (!source->getInputs().empty())
|
||||
throw Exception("Source for pipe shouldn't have any input, but " + source->getName() + " has " +
|
||||
toString(source->getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Source for pipe shouldn't have any input, but {} has {} inputs",
|
||||
source->getName(),
|
||||
source->getInputs().size());
|
||||
|
||||
if (!output)
|
||||
throw Exception("Cannot create Pipe from source because specified output port is nullptr",
|
||||
@ -141,8 +146,7 @@ Pipe::Pipe(ProcessorPtr source, OutputPort * output, OutputPort * totals, Output
|
||||
|
||||
auto it = std::find_if(outputs.begin(), outputs.end(), [port](const OutputPort & p) { return &p == port; });
|
||||
if (it == outputs.end())
|
||||
throw Exception("Cannot create Pipe because specified " + name + " port does not belong to source",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create Pipe because specified {} port does not belong to source", name);
|
||||
};
|
||||
|
||||
check_port_from_source(output, "output");
|
||||
@ -150,9 +154,11 @@ Pipe::Pipe(ProcessorPtr source, OutputPort * output, OutputPort * totals, Output
|
||||
check_port_from_source(extremes, "extremes");
|
||||
|
||||
if (num_specified_ports != outputs.size())
|
||||
throw Exception("Cannot create Pipe from source because it has " + std::to_string(outputs.size()) +
|
||||
" output ports, but " + std::to_string(num_specified_ports) + " were specified",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot create Pipe from source because it has {} output ports, but {} were specified",
|
||||
outputs.size(),
|
||||
num_specified_ports);
|
||||
}
|
||||
|
||||
totals_port = totals;
|
||||
@ -188,14 +194,16 @@ Pipe::Pipe(Processors processors_) : processors(std::move(processors_))
|
||||
for (const auto & port : processor->getInputs())
|
||||
{
|
||||
if (!port.isConnected())
|
||||
throw Exception("Cannot create Pipe because processor " + processor->getName() +
|
||||
" has not connected input port", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR, "Cannot create Pipe because processor {} has disconnected input port", processor->getName());
|
||||
|
||||
const auto * connected_processor = &port.getOutputPort().getProcessor();
|
||||
if (!set.contains(connected_processor))
|
||||
throw Exception("Cannot create Pipe because processor " + processor->getName() +
|
||||
" has input port which is connected with unknown processor " +
|
||||
connected_processor->getName(), ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot create Pipe because processor {} has input port which is connected with unknown processor {}",
|
||||
processor->getName(),
|
||||
connected_processor->getName());
|
||||
}
|
||||
|
||||
for (auto & port : processor->getOutputs())
|
||||
@ -208,14 +216,16 @@ Pipe::Pipe(Processors processors_) : processors(std::move(processors_))
|
||||
|
||||
const auto * connected_processor = &port.getInputPort().getProcessor();
|
||||
if (!set.contains(connected_processor))
|
||||
throw Exception("Cannot create Pipe because processor " + processor->getName() +
|
||||
" has output port which is connected with unknown processor " +
|
||||
connected_processor->getName(), ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot create Pipe because processor {} has output port which is connected with unknown processor {}",
|
||||
processor->getName(),
|
||||
connected_processor->getName());
|
||||
}
|
||||
}
|
||||
|
||||
if (output_ports.empty())
|
||||
throw Exception("Cannot create Pipe because processors don't have any not-connected output ports",
|
||||
throw Exception("Cannot create Pipe because processors don't have any disconnected output ports",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
header = output_ports.front()->getHeader();
|
||||
@ -365,10 +375,10 @@ void Pipe::addSource(ProcessorPtr source)
|
||||
void Pipe::addTotalsSource(ProcessorPtr source)
|
||||
{
|
||||
if (output_ports.empty())
|
||||
throw Exception("Cannot add totals source to empty Pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot add totals source to empty Pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (totals_port)
|
||||
throw Exception("Totals source was already added to Pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Totals source was already added to Pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
checkSource(*source);
|
||||
const auto & source_header = output_ports.front()->getHeader();
|
||||
@ -385,10 +395,10 @@ void Pipe::addTotalsSource(ProcessorPtr source)
|
||||
void Pipe::addExtremesSource(ProcessorPtr source)
|
||||
{
|
||||
if (output_ports.empty())
|
||||
throw Exception("Cannot add extremes source to empty Pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot add extremes source to empty Pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (extremes_port)
|
||||
throw Exception("Extremes source was already added to Pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Extremes source was already added to Pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
checkSource(*source);
|
||||
const auto & source_header = output_ports.front()->getHeader();
|
||||
@ -435,20 +445,23 @@ void Pipe::addTransform(ProcessorPtr transform)
|
||||
void Pipe::addTransform(ProcessorPtr transform, OutputPort * totals, OutputPort * extremes)
|
||||
{
|
||||
if (output_ports.empty())
|
||||
throw Exception("Cannot add transform to empty Pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot add transform to empty Pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
auto & inputs = transform->getInputs();
|
||||
if (inputs.size() != output_ports.size())
|
||||
throw Exception("Cannot add transform " + transform->getName() + " to Pipes because "
|
||||
"Processor has " + std::to_string(inputs.size()) + " input ports, "
|
||||
"but " + std::to_string(output_ports.size()) + " expected", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add transform {} to Pipe because it has {} input ports, but {} expected",
|
||||
transform->getName(),
|
||||
inputs.size(),
|
||||
output_ports.size());
|
||||
|
||||
if (totals && totals_port)
|
||||
throw Exception("Cannot add transform with totals to Pipe because it already has totals.",
|
||||
throw Exception("Cannot add transform with totals to Pipe because it already has totals",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (extremes && extremes_port)
|
||||
throw Exception("Cannot add transform with extremes to Pipe because it already has extremes.",
|
||||
throw Exception("Cannot add transform with extremes to Pipe because it already has extremes",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (totals)
|
||||
@ -515,21 +528,24 @@ void Pipe::addTransform(ProcessorPtr transform, OutputPort * totals, OutputPort
|
||||
void Pipe::addTransform(ProcessorPtr transform, InputPort * totals, InputPort * extremes)
|
||||
{
|
||||
if (output_ports.empty())
|
||||
throw Exception("Cannot add transform to empty Pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot add transform to empty Pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
auto & inputs = transform->getInputs();
|
||||
size_t expected_inputs = output_ports.size() + (totals ? 1 : 0) + (extremes ? 1 : 0);
|
||||
if (inputs.size() != expected_inputs)
|
||||
throw Exception("Cannot add transform " + transform->getName() + " to Pipes because "
|
||||
"Processor has " + std::to_string(inputs.size()) + " input ports, "
|
||||
"but " + std::to_string(expected_inputs) + " expected", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add transform {} to Pipe because it has {} input ports, but {} expected",
|
||||
transform->getName(),
|
||||
inputs.size(),
|
||||
expected_inputs);
|
||||
|
||||
if (totals && !totals_port)
|
||||
throw Exception("Cannot add transform consuming totals to Pipe because Pipe does not have totals.",
|
||||
throw Exception("Cannot add transform consuming totals to Pipe because Pipe does not have totals",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (extremes && !extremes_port)
|
||||
throw Exception("Cannot add transform consuming extremes to Pipe because it already has extremes.",
|
||||
throw Exception("Cannot add transform consuming extremes to Pipe because it already has extremes",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (totals)
|
||||
@ -561,17 +577,20 @@ void Pipe::addTransform(ProcessorPtr transform, InputPort * totals, InputPort *
|
||||
}
|
||||
|
||||
if (totals && !found_totals)
|
||||
throw Exception("Cannot add transform " + transform->getName() + " to Pipes because "
|
||||
"specified totals port does not belong to it", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add transform {} to Pipes because specified totals port does not belong to it",
|
||||
transform->getName());
|
||||
|
||||
if (extremes && !found_extremes)
|
||||
throw Exception("Cannot add transform " + transform->getName() + " to Pipes because "
|
||||
"specified extremes port does not belong to it", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add transform {} to Pipes because specified extremes port does not belong to it",
|
||||
transform->getName());
|
||||
|
||||
auto & outputs = transform->getOutputs();
|
||||
if (outputs.empty())
|
||||
throw Exception("Cannot add transform " + transform->getName() + " to Pipes because it has no outputs",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot add transform {} to Pipes because it has no outputs", transform->getName());
|
||||
|
||||
output_ports.clear();
|
||||
output_ports.reserve(outputs.size());
|
||||
@ -614,14 +633,18 @@ void Pipe::addSimpleTransform(const ProcessorGetterWithStreamKind & getter)
|
||||
if (transform)
|
||||
{
|
||||
if (transform->getInputs().size() != 1)
|
||||
throw Exception("Processor for query pipeline transform should have single input, "
|
||||
"but " + transform->getName() + " has " +
|
||||
toString(transform->getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Processor for query pipeline transform should have single input, but {} has {} inputs",
|
||||
transform->getName(),
|
||||
transform->getInputs().size());
|
||||
|
||||
if (transform->getOutputs().size() != 1)
|
||||
throw Exception("Processor for query pipeline transform should have single output, "
|
||||
"but " + transform->getName() + " has " +
|
||||
toString(transform->getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Processor for query pipeline transform should have single output, but {} has {} outputs",
|
||||
transform->getName(),
|
||||
transform->getOutputs().size());
|
||||
}
|
||||
|
||||
const auto & out_header = transform ? transform->getOutputs().front().getHeader()
|
||||
@ -661,10 +684,11 @@ void Pipe::addSimpleTransform(const ProcessorGetter & getter)
|
||||
void Pipe::addChains(std::vector<Chain> chains)
|
||||
{
|
||||
if (output_ports.size() != chains.size())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add chains to Pipe because "
|
||||
"number of output ports ({}) is not equal to the number of chains ({})",
|
||||
output_ports.size(), chains.size());
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot add chains to Pipe because number of output ports ({}) is not equal to the number of chains ({})",
|
||||
output_ports.size(),
|
||||
chains.size());
|
||||
|
||||
dropTotals();
|
||||
dropExtremes();
|
||||
@ -702,7 +726,7 @@ void Pipe::addChains(std::vector<Chain> chains)
|
||||
void Pipe::resize(size_t num_streams, bool force, bool strict)
|
||||
{
|
||||
if (output_ports.empty())
|
||||
throw Exception("Cannot resize an empty Pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot resize an empty Pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (!force && num_streams == numOutputPorts())
|
||||
return;
|
||||
@ -720,7 +744,7 @@ void Pipe::resize(size_t num_streams, bool force, bool strict)
|
||||
void Pipe::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter)
|
||||
{
|
||||
if (output_ports.empty())
|
||||
throw Exception("Cannot set sink to empty Pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot set sink to empty Pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
auto add_transform = [&](OutputPort *& stream, Pipe::StreamType stream_type)
|
||||
{
|
||||
@ -732,14 +756,18 @@ void Pipe::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter)
|
||||
if (transform)
|
||||
{
|
||||
if (transform->getInputs().size() != 1)
|
||||
throw Exception("Sink for query pipeline transform should have single input, "
|
||||
"but " + transform->getName() + " has " +
|
||||
toString(transform->getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Sink for query pipeline transform should have single input, but {} has {} inputs",
|
||||
transform->getName(),
|
||||
transform->getInputs().size());
|
||||
|
||||
if (!transform->getOutputs().empty())
|
||||
throw Exception("Sink for query pipeline transform should have no outputs, "
|
||||
"but " + transform->getName() + " has " +
|
||||
toString(transform->getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Sink for query pipeline transform should have no outputs, but {} has {} outputs",
|
||||
transform->getName(),
|
||||
transform->getOutputs().size());
|
||||
}
|
||||
|
||||
if (!transform)
|
||||
@ -762,7 +790,7 @@ void Pipe::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter)
|
||||
void Pipe::transform(const Transformer & transformer)
|
||||
{
|
||||
if (output_ports.empty())
|
||||
throw Exception("Cannot transform empty Pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot transform empty Pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
auto new_processors = transformer(output_ports);
|
||||
|
||||
@ -774,8 +802,10 @@ void Pipe::transform(const Transformer & transformer)
|
||||
for (const auto & port : output_ports)
|
||||
{
|
||||
if (!port->isConnected())
|
||||
throw Exception("Transformation of Pipe is not valid because output port (" +
|
||||
port->getHeader().dumpStructure() + ") is not connected", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Transformation of Pipe is not valid because output port ({})",
|
||||
port->getHeader().dumpStructure());
|
||||
|
||||
set.emplace(&port->getProcessor());
|
||||
}
|
||||
@ -787,14 +817,18 @@ void Pipe::transform(const Transformer & transformer)
|
||||
for (const auto & port : processor->getInputs())
|
||||
{
|
||||
if (!port.isConnected())
|
||||
throw Exception("Transformation of Pipe is not valid because processor " + processor->getName() +
|
||||
" has not connected input port", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Transformation of Pipe is not valid because processor {} has not connected input port",
|
||||
processor->getName());
|
||||
|
||||
const auto * connected_processor = &port.getOutputPort().getProcessor();
|
||||
if (!set.contains(connected_processor))
|
||||
throw Exception("Transformation of Pipe is not valid because processor " + processor->getName() +
|
||||
" has input port which is connected with unknown processor " +
|
||||
connected_processor->getName(), ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Transformation of Pipe is not valid because processor {} has input port which is connected with unknown processor {}",
|
||||
processor->getName(),
|
||||
connected_processor->getName());
|
||||
}
|
||||
|
||||
for (auto & port : processor->getOutputs())
|
||||
@ -807,15 +841,17 @@ void Pipe::transform(const Transformer & transformer)
|
||||
|
||||
const auto * connected_processor = &port.getInputPort().getProcessor();
|
||||
if (!set.contains(connected_processor))
|
||||
throw Exception("Transformation of Pipe is not valid because processor " + processor->getName() +
|
||||
" has output port which is connected with unknown processor " +
|
||||
connected_processor->getName(), ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Transformation of Pipe is not valid because processor {} has output port which is connected with unknown processor {}",
|
||||
processor->getName(),
|
||||
connected_processor->getName());
|
||||
}
|
||||
}
|
||||
|
||||
if (output_ports.empty())
|
||||
throw Exception("Transformation of Pipe is not valid because processors don't have any "
|
||||
"not-connected output ports", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
"Transformation of Pipe is not valid because processors don't have any disconnected output ports", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
header = output_ports.front()->getHeader();
|
||||
for (size_t i = 1; i < output_ports.size(); ++i)
|
||||
|
||||
@ -18,7 +18,7 @@ struct PipelineResourcesHolder
|
||||
PipelineResourcesHolder();
|
||||
PipelineResourcesHolder(PipelineResourcesHolder &&) noexcept;
|
||||
~PipelineResourcesHolder();
|
||||
/// Custom mode assignment does not destroy data from lhs. It appends data from rhs to lhs.
|
||||
/// Custom move assignment does not destroy data from lhs. It appends data from rhs to lhs.
|
||||
PipelineResourcesHolder& operator=(PipelineResourcesHolder &&) noexcept;
|
||||
|
||||
/// Some processors may implicitly use Context or temporary Storage created by Interpreter.
|
||||
|
||||
@ -33,7 +33,7 @@ static void checkInput(const InputPort & input, const ProcessorPtr & processor)
|
||||
if (!input.isConnected())
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot create QueryPipeline because {} has not connected input",
|
||||
"Cannot create QueryPipeline because {} has disconnected input",
|
||||
processor->getName());
|
||||
}
|
||||
|
||||
@ -42,7 +42,7 @@ static void checkOutput(const OutputPort & output, const ProcessorPtr & processo
|
||||
if (!output.isConnected())
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Cannot create QueryPipeline because {} has not connected output",
|
||||
"Cannot create QueryPipeline because {} has disconnected output",
|
||||
processor->getName());
|
||||
}
|
||||
|
||||
|
||||
@ -37,7 +37,7 @@ void QueryPipelineBuilder::addQueryPlan(std::unique_ptr<QueryPlan> plan)
|
||||
void QueryPipelineBuilder::checkInitialized()
|
||||
{
|
||||
if (!initialized())
|
||||
throw Exception("QueryPipeline wasn't initialized.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("QueryPipeline is uninitialized", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
void QueryPipelineBuilder::checkInitializedAndNotCompleted()
|
||||
@ -45,35 +45,44 @@ void QueryPipelineBuilder::checkInitializedAndNotCompleted()
|
||||
checkInitialized();
|
||||
|
||||
if (pipe.isCompleted())
|
||||
throw Exception("QueryPipeline was already completed.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("QueryPipeline is already completed", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
static void checkSource(const ProcessorPtr & source, bool can_have_totals)
|
||||
{
|
||||
if (!source->getInputs().empty())
|
||||
throw Exception("Source for query pipeline shouldn't have any input, but " + source->getName() + " has " +
|
||||
toString(source->getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Source for query pipeline shouldn't have any input, but {} has {} inputs",
|
||||
source->getName(),
|
||||
source->getInputs().size());
|
||||
|
||||
if (source->getOutputs().empty())
|
||||
throw Exception("Source for query pipeline should have single output, but it doesn't have any",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR, "Source for query pipeline should have single output, but {} doesn't have any", source->getName());
|
||||
|
||||
if (!can_have_totals && source->getOutputs().size() != 1)
|
||||
throw Exception("Source for query pipeline should have single output, but " + source->getName() + " has " +
|
||||
toString(source->getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Source for query pipeline should have single output, but {} has {} outputs",
|
||||
source->getName(),
|
||||
source->getOutputs().size());
|
||||
|
||||
if (source->getOutputs().size() > 2)
|
||||
throw Exception("Source for query pipeline should have 1 or 2 outputs, but " + source->getName() + " has " +
|
||||
toString(source->getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Source for query pipeline should have 1 or 2 output, but {} has {} outputs",
|
||||
source->getName(),
|
||||
source->getOutputs().size());
|
||||
}
|
||||
|
||||
void QueryPipelineBuilder::init(Pipe pipe_)
|
||||
{
|
||||
if (initialized())
|
||||
throw Exception("Pipeline has already been initialized.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Pipeline has already been initialized", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (pipe_.empty())
|
||||
throw Exception("Can't initialize pipeline with empty pipe.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Can't initialize pipeline with empty pipe", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
pipe = std::move(pipe_);
|
||||
}
|
||||
@ -81,10 +90,10 @@ void QueryPipelineBuilder::init(Pipe pipe_)
|
||||
void QueryPipelineBuilder::init(QueryPipeline pipeline)
|
||||
{
|
||||
if (initialized())
|
||||
throw Exception("Pipeline has already been initialized.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Pipeline has already been initialized", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (pipeline.pushing())
|
||||
throw Exception("Can't initialize pushing pipeline.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Can't initialize pushing pipeline", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
pipe.holder = std::move(pipeline.resources);
|
||||
pipe.processors = std::move(pipeline.processors);
|
||||
@ -191,11 +200,10 @@ void QueryPipelineBuilder::addTotalsHavingTransform(ProcessorPtr transform)
|
||||
checkInitializedAndNotCompleted();
|
||||
|
||||
if (!typeid_cast<const TotalsHavingTransform *>(transform.get()))
|
||||
throw Exception("TotalsHavingTransform expected for QueryPipeline::addTotalsHavingTransform.",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("TotalsHavingTransform is expected for QueryPipeline::addTotalsHavingTransform", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (pipe.getTotalsPort())
|
||||
throw Exception("Totals having transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Totals having transform was already added to pipeline", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
resize(1);
|
||||
|
||||
@ -208,7 +216,7 @@ void QueryPipelineBuilder::addDefaultTotals()
|
||||
checkInitializedAndNotCompleted();
|
||||
|
||||
if (pipe.getTotalsPort())
|
||||
throw Exception("Totals having transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Totals having transform was already added to pipeline", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
const auto & current_header = getHeader();
|
||||
Columns columns;
|
||||
@ -461,7 +469,7 @@ void QueryPipelineBuilder::setProcessListElement(QueryStatus * elem)
|
||||
PipelineExecutorPtr QueryPipelineBuilder::execute()
|
||||
{
|
||||
if (!isCompleted())
|
||||
throw Exception("Cannot execute pipeline because it is not completed.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot execute pipeline because it is not completed", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
return std::make_shared<PipelineExecutor>(pipe.processors, process_list_element);
|
||||
}
|
||||
|
||||
@ -78,8 +78,10 @@ RemoteInserter::RemoteInserter(
|
||||
/// client's already got this information for remote table. Ignore.
|
||||
}
|
||||
else
|
||||
throw NetException("Unexpected packet from server (expected Data or Exception, got "
|
||||
+ String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
|
||||
throw NetException(
|
||||
ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER,
|
||||
"Unexpected packet from server (expected Data or Exception, got {})",
|
||||
Protocol::Server::toString(packet.type));
|
||||
}
|
||||
}
|
||||
|
||||
@ -131,8 +133,10 @@ void RemoteInserter::onFinish()
|
||||
// Do nothing
|
||||
}
|
||||
else
|
||||
throw NetException("Unexpected packet from server (expected EndOfStream or Exception, got "
|
||||
+ String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
|
||||
throw NetException(
|
||||
ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER,
|
||||
"Unexpected packet from server (expected EndOfStream or Exception, got {})",
|
||||
Protocol::Server::toString(packet.type));
|
||||
}
|
||||
|
||||
finished = true;
|
||||
|
||||
@ -356,7 +356,7 @@ std::variant<Block, int> RemoteQueryExecutor::restartQueryWithoutDuplicatedUUIDs
|
||||
else
|
||||
return read(*read_context);
|
||||
}
|
||||
throw Exception("Found duplicate uuids while processing query.", ErrorCodes::DUPLICATED_PART_UUIDS);
|
||||
throw Exception("Found duplicate uuids while processing query", ErrorCodes::DUPLICATED_PART_UUIDS);
|
||||
}
|
||||
|
||||
std::optional<Block> RemoteQueryExecutor::processPacket(Packet packet)
|
||||
@ -432,8 +432,10 @@ std::optional<Block> RemoteQueryExecutor::processPacket(Packet packet)
|
||||
|
||||
default:
|
||||
got_unknown_packet_from_replica = true;
|
||||
throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from one of the following replicas: {}",
|
||||
toString(packet.type),
|
||||
throw Exception(
|
||||
ErrorCodes::UNKNOWN_PACKET_FROM_SERVER,
|
||||
"Unknown packet {} from one of the following replicas: {}",
|
||||
packet.type,
|
||||
connections->dumpAddresses());
|
||||
}
|
||||
|
||||
|
||||
@ -12,12 +12,20 @@ bool SizeLimits::check(UInt64 rows, UInt64 bytes, const char * what, int too_man
|
||||
if (overflow_mode == OverflowMode::THROW)
|
||||
{
|
||||
if (max_rows && rows > max_rows)
|
||||
throw Exception("Limit for " + std::string(what) + " exceeded, max rows: " + formatReadableQuantity(max_rows)
|
||||
+ ", current rows: " + formatReadableQuantity(rows), too_many_rows_exception_code);
|
||||
throw Exception(
|
||||
too_many_rows_exception_code,
|
||||
"Limit for {} exceeded, max rows: {}, current rows: {}",
|
||||
what,
|
||||
formatReadableQuantity(max_rows),
|
||||
formatReadableQuantity(rows));
|
||||
|
||||
if (max_bytes && bytes > max_bytes)
|
||||
throw Exception(fmt::format("Limit for {} exceeded, max bytes: {}, current bytes: {}",
|
||||
std::string(what), ReadableSize(max_bytes), ReadableSize(bytes)), too_many_bytes_exception_code);
|
||||
throw Exception(
|
||||
too_many_bytes_exception_code,
|
||||
"Limit for {} exceeded, max bytes: {}, current bytes: {}",
|
||||
what,
|
||||
ReadableSize(max_bytes),
|
||||
ReadableSize(bytes));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
#include <QueryPipeline/narrowPipe.h>
|
||||
|
||||
#include <random>
|
||||
#include <Common/thread_local_rng.h>
|
||||
#include <Processors/ConcatProcessor.h>
|
||||
#include <QueryPipeline/Pipe.h>
|
||||
#include "narrowBlockInputStreams.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -9,7 +9,7 @@ namespace DB
|
||||
class Pipe;
|
||||
|
||||
/** If the number of sources of `inputs` is greater than `width`,
|
||||
* then glues the sources to each other (using ConcatBlockInputStream),
|
||||
* then glues the sources to each other (using ConcatProcessor),
|
||||
* so that the number of sources becomes no more than `width`.
|
||||
*
|
||||
* Trying to glue the sources with each other uniformly randomly.
|
||||
@ -182,41 +182,22 @@ ColumnsDescription StorageHDFS::getTableStructureFromData(
|
||||
{
|
||||
const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(uri);
|
||||
auto paths = getPathsList(path_from_uri, uri, ctx);
|
||||
if (paths.empty() && !FormatFactory::instance().checkIfFormatHasExternalSchemaReader(format))
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
"Cannot extract table structure from {} format file, because there are no files in HDFS with provided path. You must "
|
||||
"specify table structure manually",
|
||||
format);
|
||||
|
||||
std::string exception_messages;
|
||||
bool read_buffer_creator_was_used = false;
|
||||
for (const auto & path : paths)
|
||||
ReadBufferIterator read_buffer_iterator = [&, uri_without_path = uri_without_path, it = paths.begin()]() mutable -> std::unique_ptr<ReadBuffer>
|
||||
{
|
||||
auto read_buffer_creator = [&, uri_without_path = uri_without_path]()
|
||||
{
|
||||
read_buffer_creator_was_used = true;
|
||||
|
||||
if (paths.empty())
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
"Cannot extract table structure from {} format file, because there are no files in HDFS with provided path. You must "
|
||||
"specify table structure manually",
|
||||
format);
|
||||
|
||||
auto compression = chooseCompressionMethod(path, compression_method);
|
||||
return wrapReadBufferWithCompressionMethod(
|
||||
std::make_unique<ReadBufferFromHDFS>(uri_without_path, path, ctx->getGlobalContext()->getConfigRef()), compression);
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
return readSchemaFromFormat(format, std::nullopt, read_buffer_creator, ctx);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
if (paths.size() == 1 || !read_buffer_creator_was_used)
|
||||
throw;
|
||||
|
||||
exception_messages += getCurrentExceptionMessage(false) + "\n";
|
||||
}
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from hdfs files failed. Errors:\n{}", exception_messages);
|
||||
if (it == paths.end())
|
||||
return nullptr;
|
||||
auto compression = chooseCompressionMethod(*it, compression_method);
|
||||
return wrapReadBufferWithCompressionMethod(
|
||||
std::make_unique<ReadBufferFromHDFS>(uri_without_path, *it++, ctx->getGlobalContext()->getConfigRef()), compression);
|
||||
};
|
||||
return readSchemaFromFormat(format, std::nullopt, read_buffer_iterator, paths.size() > 1, ctx);
|
||||
}
|
||||
|
||||
class HDFSSource::DisclosedGlobIterator::Impl
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
#include <Interpreters/InterpreterSelectQuery.h>
|
||||
#include <Interpreters/getTableExpressions.h>
|
||||
#include <Processors/Transforms/AddingDefaultsTransform.h>
|
||||
#include <QueryPipeline/narrowBlockInputStreams.h>
|
||||
#include <QueryPipeline/narrowPipe.h>
|
||||
#include <QueryPipeline/Pipe.h>
|
||||
#include <Processors/Sources/RemoteSource.h>
|
||||
#include <QueryPipeline/RemoteQueryExecutor.h>
|
||||
|
||||
@ -5256,6 +5256,10 @@ std::optional<ProjectionCandidate> MergeTreeData::getQueryProcessingStageWithAgg
|
||||
if (select_query->join())
|
||||
return std::nullopt;
|
||||
|
||||
// INTERPOLATE expressions may include aliases, so aliases should be preserved
|
||||
if (select_query->interpolate() && !select_query->interpolate()->children.empty())
|
||||
return std::nullopt;
|
||||
|
||||
auto query_options = SelectQueryOptions(
|
||||
QueryProcessingStage::WithMergeableState,
|
||||
/* depth */ 1,
|
||||
|
||||
@ -237,7 +237,7 @@ ColumnsDescription StorageFile::getTableStructureFromFileDescriptor(ContextPtr c
|
||||
/// in case of file descriptor we have a stream of data and we cannot
|
||||
/// start reading data from the beginning after reading some data for
|
||||
/// schema inference.
|
||||
auto read_buffer_creator = [&]()
|
||||
ReadBufferIterator read_buffer_iterator = [&]()
|
||||
{
|
||||
/// We will use PeekableReadBuffer to create a checkpoint, so we need a place
|
||||
/// where we can store the original read buffer.
|
||||
@ -247,7 +247,7 @@ ColumnsDescription StorageFile::getTableStructureFromFileDescriptor(ContextPtr c
|
||||
return read_buf;
|
||||
};
|
||||
|
||||
auto columns = readSchemaFromFormat(format_name, format_settings, read_buffer_creator, context, peekable_read_buffer_from_fd);
|
||||
auto columns = readSchemaFromFormat(format_name, format_settings, read_buffer_iterator, false, context, peekable_read_buffer_from_fd);
|
||||
if (peekable_read_buffer_from_fd)
|
||||
{
|
||||
/// If we have created read buffer in readSchemaFromFormat we should rollback to checkpoint.
|
||||
@ -274,38 +274,22 @@ ColumnsDescription StorageFile::getTableStructureFromFile(
|
||||
return ColumnsDescription(source->getOutputs().front().getHeader().getNamesAndTypesList());
|
||||
}
|
||||
|
||||
std::string exception_messages;
|
||||
bool read_buffer_creator_was_used = false;
|
||||
for (const auto & path : paths)
|
||||
if (paths.empty() && !FormatFactory::instance().checkIfFormatHasExternalSchemaReader(format))
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
"Cannot extract table structure from {} format file, because there are no files with provided path. You must specify "
|
||||
"table structure manually",
|
||||
format);
|
||||
|
||||
ReadBufferIterator read_buffer_iterator = [&, it = paths.begin()]() mutable -> std::unique_ptr<ReadBuffer>
|
||||
{
|
||||
auto read_buffer_creator = [&]()
|
||||
{
|
||||
read_buffer_creator_was_used = true;
|
||||
if (it == paths.end())
|
||||
return nullptr;
|
||||
|
||||
if (!std::filesystem::exists(path))
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
"Cannot extract table structure from {} format file, because there are no files with provided path. You must specify "
|
||||
"table structure manually",
|
||||
format);
|
||||
return createReadBuffer(*it++, false, "File", -1, compression_method, context);
|
||||
};
|
||||
|
||||
return createReadBuffer(path, false, "File", -1, compression_method, context);
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
return readSchemaFromFormat(format, format_settings, read_buffer_creator, context);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
if (paths.size() == 1 || !read_buffer_creator_was_used)
|
||||
throw;
|
||||
|
||||
exception_messages += getCurrentExceptionMessage(false) + "\n";
|
||||
}
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from files failed. Errors:\n{}", exception_messages);
|
||||
return readSchemaFromFormat(format, format_settings, read_buffer_iterator, paths.size() > 1, context);
|
||||
}
|
||||
|
||||
bool StorageFile::isColumnOriented() const
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include <QueryPipeline/narrowBlockInputStreams.h>
|
||||
#include <QueryPipeline/narrowPipe.h>
|
||||
#include <Processors/Sources/SourceFromSingleChunk.h>
|
||||
#include <Storages/StorageMerge.h>
|
||||
#include <Storages/StorageFactory.h>
|
||||
|
||||
@ -36,7 +36,7 @@
|
||||
#include <Processors/Transforms/AddingDefaultsTransform.h>
|
||||
#include <Processors/Formats/IOutputFormat.h>
|
||||
#include <Processors/Formats/IInputFormat.h>
|
||||
#include <QueryPipeline/narrowBlockInputStreams.h>
|
||||
#include <QueryPipeline/narrowPipe.h>
|
||||
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
#include <Processors/Executors/PullingPipelineExecutor.h>
|
||||
@ -946,45 +946,31 @@ ColumnsDescription StorageS3::getTableStructureFromDataImpl(
|
||||
const std::optional<FormatSettings> & format_settings,
|
||||
ContextPtr ctx)
|
||||
{
|
||||
std::vector<String> keys = {s3_configuration.uri.key};
|
||||
auto file_iterator = createFileIterator(s3_configuration, keys, is_key_with_globs, distributed_processing, ctx);
|
||||
auto file_iterator = createFileIterator(s3_configuration, {s3_configuration.uri.key}, is_key_with_globs, distributed_processing, ctx);
|
||||
|
||||
std::string current_key;
|
||||
std::string exception_messages;
|
||||
bool read_buffer_creator_was_used = false;
|
||||
do
|
||||
ReadBufferIterator read_buffer_iterator = [&, first = false]() mutable -> std::unique_ptr<ReadBuffer>
|
||||
{
|
||||
current_key = (*file_iterator)();
|
||||
auto read_buffer_creator = [&]()
|
||||
auto key = (*file_iterator)();
|
||||
if (key.empty())
|
||||
{
|
||||
read_buffer_creator_was_used = true;
|
||||
if (current_key.empty())
|
||||
if (first)
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
"Cannot extract table structure from {} format file, because there are no files with provided path in S3. You must specify "
|
||||
"table structure manually",
|
||||
format);
|
||||
|
||||
return wrapReadBufferWithCompressionMethod(
|
||||
std::make_unique<ReadBufferFromS3>(
|
||||
s3_configuration.client, s3_configuration.uri.bucket, current_key, s3_configuration.rw_settings.max_single_read_retries, ctx->getReadSettings()),
|
||||
chooseCompressionMethod(current_key, compression_method));
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
return readSchemaFromFormat(format, format_settings, read_buffer_creator, ctx);
|
||||
return nullptr;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
if (!is_key_with_globs || !read_buffer_creator_was_used)
|
||||
throw;
|
||||
|
||||
exception_messages += getCurrentExceptionMessage(false) + "\n";
|
||||
}
|
||||
} while (!current_key.empty());
|
||||
first = false;
|
||||
return wrapReadBufferWithCompressionMethod(
|
||||
std::make_unique<ReadBufferFromS3>(
|
||||
s3_configuration.client, s3_configuration.uri.bucket, key, s3_configuration.rw_settings.max_single_read_retries, ctx->getReadSettings()),
|
||||
chooseCompressionMethod(key, compression_method));
|
||||
};
|
||||
|
||||
throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from s3 files failed. Errors:\n{}", exception_messages);
|
||||
return readSchemaFromFormat(format, format_settings, read_buffer_iterator, is_key_with_globs, ctx);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
#include <Interpreters/InterpreterSelectQuery.h>
|
||||
#include <Interpreters/getTableExpressions.h>
|
||||
#include <Processors/Transforms/AddingDefaultsTransform.h>
|
||||
#include <QueryPipeline/narrowBlockInputStreams.h>
|
||||
#include <QueryPipeline/narrowPipe.h>
|
||||
#include <QueryPipeline/Pipe.h>
|
||||
#include "Processors/Sources/SourceWithProgress.h"
|
||||
#include <Processors/Sources/RemoteSource.h>
|
||||
|
||||
@ -43,7 +43,6 @@ namespace ErrorCodes
|
||||
extern const int NETWORK_ERROR;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
|
||||
}
|
||||
|
||||
|
||||
@ -551,49 +550,31 @@ ColumnsDescription IStorageURLBase::getTableStructureFromData(
|
||||
urls_to_check = {uri};
|
||||
}
|
||||
|
||||
String exception_messages;
|
||||
bool read_buffer_creator_was_used = false;
|
||||
|
||||
std::vector<String>::const_iterator option = urls_to_check.begin();
|
||||
do
|
||||
ReadBufferIterator read_buffer_iterator = [&, it = urls_to_check.cbegin()]() mutable -> std::unique_ptr<ReadBuffer>
|
||||
{
|
||||
auto read_buffer_creator = [&]()
|
||||
{
|
||||
read_buffer_creator_was_used = true;
|
||||
return StorageURLSource::getFirstAvailableURLReadBuffer(
|
||||
option,
|
||||
urls_to_check.end(),
|
||||
context,
|
||||
{},
|
||||
Poco::Net::HTTPRequest::HTTP_GET,
|
||||
{},
|
||||
ConnectionTimeouts::getHTTPTimeouts(context),
|
||||
compression_method,
|
||||
credentials,
|
||||
headers,
|
||||
false,
|
||||
false,
|
||||
context->getSettingsRef().max_download_threads);
|
||||
};
|
||||
if (it == urls_to_check.cend())
|
||||
return nullptr;
|
||||
|
||||
try
|
||||
{
|
||||
return readSchemaFromFormat(format, format_settings, read_buffer_creator, context);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
if (urls_to_check.size() == 1 || !read_buffer_creator_was_used)
|
||||
throw;
|
||||
auto buf = StorageURLSource::getFirstAvailableURLReadBuffer(
|
||||
it,
|
||||
urls_to_check.cend(),
|
||||
context,
|
||||
{},
|
||||
Poco::Net::HTTPRequest::HTTP_GET,
|
||||
{},
|
||||
ConnectionTimeouts::getHTTPTimeouts(context),
|
||||
compression_method,
|
||||
credentials,
|
||||
headers,
|
||||
false,
|
||||
false,
|
||||
context->getSettingsRef().max_download_threads);\
|
||||
++it;
|
||||
return buf;
|
||||
};
|
||||
|
||||
exception_messages += getCurrentExceptionMessage(false) + "\n";
|
||||
}
|
||||
|
||||
} while (++option < urls_to_check.end());
|
||||
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
|
||||
"All attempts to extract table structure from urls failed. Errors:\n{}",
|
||||
exception_messages);
|
||||
return readSchemaFromFormat(format, format_settings, read_buffer_iterator, urls_to_check.size() > 1, context);
|
||||
}
|
||||
|
||||
bool IStorageURLBase::isColumnOriented() const
|
||||
|
||||
@ -49,12 +49,11 @@ void TableFunctionFormat::parseArguments(const ASTPtr & ast_function, ContextPtr
|
||||
|
||||
ColumnsDescription TableFunctionFormat::getActualTableStructure(ContextPtr context) const
|
||||
{
|
||||
auto read_buffer_creator = [&]()
|
||||
ReadBufferIterator read_buffer_iterator = [&]()
|
||||
{
|
||||
return std::make_unique<ReadBufferFromString>(data);
|
||||
};
|
||||
|
||||
return readSchemaFromFormat(format, std::nullopt, read_buffer_creator, context);
|
||||
return readSchemaFromFormat(format, std::nullopt, read_buffer_iterator, false, context);
|
||||
}
|
||||
|
||||
Block TableFunctionFormat::parseData(ColumnsDescription columns, ContextPtr context) const
|
||||
|
||||
@ -61,6 +61,8 @@ set the following environment variables:
|
||||
* `CLICKHOUSE_TESTS_CLIENT_BIN_PATH` to choose the client binary.
|
||||
* `CLICKHOUSE_TESTS_BASE_CONFIG_DIR` to choose the directory from which base configs (`config.xml` and`users.xml`) are taken.
|
||||
|
||||
Please note that if you use separate build (`ENABLE_CLICKHOUSE_ALL=OFF`), you need to build different components, including but not limited to `ENABLE_CLICKHOUSE_LIBRARY_BRIDGE=ON ENABLE_CLICKHOUSE_ODBC_BRIDGE=ON ENABLE_CLICKHOUSE_KEEPER=ON`. So it is easier to use `ENABLE_CLICKHOUSE_ALL=ON`
|
||||
|
||||
For tests that use common docker compose files you may need to set up their path with environment variable: `DOCKER_COMPOSE_DIR=$HOME/ClickHouse/docker/test/integration/runner/compose`
|
||||
|
||||
### Running with runner script
|
||||
|
||||
@ -541,6 +541,30 @@ def test_schema_inference_with_globs(started_cluster):
|
||||
)
|
||||
assert sorted(result.split()) == ["0", "\\N"]
|
||||
|
||||
node1.query(
|
||||
f"insert into table function hdfs('hdfs://hdfs1:9000/data3.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL"
|
||||
)
|
||||
|
||||
filename = "data{1,3}.jsoncompacteachrow"
|
||||
|
||||
result = node1.query_and_get_error(f"desc hdfs('hdfs://hdfs1:9000/{filename}')")
|
||||
|
||||
assert "All attempts to extract table structure from files failed" in result
|
||||
|
||||
node1.query(
|
||||
f"insert into table function hdfs('hdfs://hdfs1:9000/data0.jsoncompacteachrow', 'TSV', 'x String') select '[123;]'"
|
||||
)
|
||||
|
||||
url_filename = "data{0,1,2,3}.jsoncompacteachrow"
|
||||
|
||||
result = node1.query_and_get_error(
|
||||
f"desc hdfs('hdfs://hdfs1:9000/data*.jsoncompacteachrow')"
|
||||
)
|
||||
|
||||
assert (
|
||||
"Cannot extract table structure from JSONCompactEachRow format file" in result
|
||||
)
|
||||
|
||||
|
||||
def test_insert_select_schema_inference(started_cluster):
|
||||
node1.query(
|
||||
|
||||
@ -697,6 +697,11 @@ def test_abrupt_connection_loss_while_heavy_replication(started_cluster):
|
||||
|
||||
|
||||
def test_abrupt_server_restart_while_heavy_replication(started_cluster):
|
||||
|
||||
# FIXME (kssenii) temporary disabled
|
||||
if instance.is_built_with_address_sanitizer():
|
||||
pytest.skip("Temporary disabled (FIXME)")
|
||||
|
||||
conn = get_postgres_conn(
|
||||
ip=started_cluster.postgres_ip,
|
||||
port=started_cluster.postgres_port,
|
||||
|
||||
@ -1325,6 +1325,46 @@ def test_schema_inference_from_globs(started_cluster):
|
||||
)
|
||||
assert sorted(result.split()) == ["0", "\\N"]
|
||||
|
||||
instance.query(
|
||||
f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test3.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL"
|
||||
)
|
||||
|
||||
url_filename = "test{1,3}.jsoncompacteachrow"
|
||||
|
||||
result = instance.query_and_get_error(
|
||||
f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}')"
|
||||
)
|
||||
|
||||
assert "All attempts to extract table structure from files failed" in result
|
||||
|
||||
result = instance.query_and_get_error(
|
||||
f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}')"
|
||||
)
|
||||
|
||||
assert "All attempts to extract table structure from files failed" in result
|
||||
|
||||
instance.query(
|
||||
f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test0.jsoncompacteachrow', 'TSV', 'x String') select '[123;]'"
|
||||
)
|
||||
|
||||
result = instance.query_and_get_error(
|
||||
f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test*.jsoncompacteachrow')"
|
||||
)
|
||||
|
||||
assert (
|
||||
"Cannot extract table structure from JSONCompactEachRow format file" in result
|
||||
)
|
||||
|
||||
url_filename = "test{0,1,2,3}.jsoncompacteachrow"
|
||||
|
||||
result = instance.query_and_get_error(
|
||||
f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}')"
|
||||
)
|
||||
|
||||
assert (
|
||||
"Cannot extract table structure from JSONCompactEachRow format file" in result
|
||||
)
|
||||
|
||||
|
||||
def test_signatures(started_cluster):
|
||||
bucket = started_cluster.minio_bucket
|
||||
|
||||
@ -31,7 +31,7 @@
|
||||
|
||||
<query>
|
||||
INSERT INTO FUNCTION file('test_file', '{format}', 'key UInt64, value UInt64')
|
||||
SELECT number, number FROM numbers(1000000)
|
||||
SELECT number, number FROM numbers(10000000)
|
||||
</query>
|
||||
|
||||
<query>
|
||||
@ -39,16 +39,18 @@
|
||||
SELECT number, number, number, number, number, number FROM numbers(1000000)
|
||||
</query>
|
||||
|
||||
<!-- Disabled in 36538, broken now
|
||||
<query>
|
||||
INSERT INTO FUNCTION file('test_file_{{_partition_id}}', '{format}', 'partition_id UInt64, value UInt64')
|
||||
PARTITION BY partition_id
|
||||
SELECT (number % {partitions_count}) as partition_id, number FROM numbers(1000000)
|
||||
SELECT (number % {partitions_count}) as partition_id, number FROM numbers(50000)
|
||||
</query>
|
||||
|
||||
<query>
|
||||
INSERT INTO FUNCTION file('test_file_{{_partition_id}}', '{format}', 'partition_id UInt64, value1 UInt64, value2 UInt64, value3 UInt64, value4 UInt64, value5 UInt64')
|
||||
PARTITION BY partition_id
|
||||
SELECT (number % {partitions_count}) as partition_id, number, number, number, number, number FROM numbers(1000000)
|
||||
SELECT (number % {partitions_count}) as partition_id, number, number, number, number, number FROM numbers(50000)
|
||||
</query>
|
||||
-->
|
||||
|
||||
</test>
|
||||
|
||||
@ -238,3 +238,9 @@ original 7
|
||||
10.5 \N
|
||||
11 \N
|
||||
11.5 \N
|
||||
1 1
|
||||
2 2
|
||||
3 2
|
||||
4 3
|
||||
5 4
|
||||
6 3
|
||||
|
||||
@ -70,3 +70,10 @@ SELECT n, source, inter + NULL AS inter_p FROM (
|
||||
SELECT n, source, inter AS inter_p FROM (
|
||||
SELECT toFloat32(number % 10) AS n, 'original' AS source, number + NULL AS inter FROM numbers(10) WHERE (number % 3) = 1
|
||||
) ORDER BY n ASC WITH FILL FROM 0 TO 11.51 STEP 0.5 INTERPOLATE ( inter_p AS inter_p + 1 );
|
||||
|
||||
# Test INTERPOLATE for MergeTree
|
||||
DROP TABLE IF EXISTS t_inter_02233;
|
||||
CREATE TABLE t_inter_02233 (n Int32) ENGINE = MergeTree ORDER BY n;
|
||||
INSERT INTO t_inter_02233 VALUES (1),(3),(3),(6),(6),(6);
|
||||
SELECT n, count() AS m FROM t_inter_02233 GROUP BY n ORDER BY n WITH FILL INTERPOLATE ( m AS m + 1 );
|
||||
DROP TABLE IF EXISTS t_inter_02233;
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
1
|
||||
\N
|
||||
@ -0,0 +1,10 @@
|
||||
-- Tags: no-fasttest, no-parallel
|
||||
insert into function file('02267_data2.jsonl') select NULL as x;
|
||||
insert into function file('02267_data3.jsonl') select * from numbers(0);
|
||||
insert into function file('02267_data4.jsonl') select 1 as x;
|
||||
select * from file('02267_data*.jsonl') order by x;
|
||||
|
||||
insert into function file('02267_data1.jsonl', 'TSV') select 1 as x;
|
||||
insert into function file('02267_data1.jsonl', 'TSV') select [1,2,3] as x;
|
||||
|
||||
select * from file('02267_data*.jsonl'); --{serverError CANNOT_EXTRACT_TABLE_STRUCTURE}
|
||||
@ -1,3 +1,4 @@
|
||||
-- Tags: no-backward-compatibility-check:22.4.2
|
||||
DROP TABLE IF EXISTS github_events;
|
||||
|
||||
CREATE TABLE github_events
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
4
tests/queries/0_stateless/02286_convert_decimal_type.sql
Normal file
4
tests/queries/0_stateless/02286_convert_decimal_type.sql
Normal file
@ -0,0 +1,4 @@
|
||||
SELECT * FROM VALUES('x Decimal32(0)', (1));
|
||||
SELECT * FROM VALUES('x Decimal64(0)', (2));
|
||||
SELECT * FROM VALUES('x Decimal128(0)', (3));
|
||||
SELECT * FROM VALUES('x Decimal256(0)', (4));
|
||||
Loading…
Reference in New Issue
Block a user