mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Compare commits
59 Commits
b446a92e8b
...
20b25566f5
Author | SHA1 | Date | |
---|---|---|---|
|
20b25566f5 | ||
|
f36408a666 | ||
|
b42c6491e4 | ||
|
1a4c7b7c61 | ||
|
4c4a051d5e | ||
|
a55cc03973 | ||
|
37411bf240 | ||
|
733c57dae7 | ||
|
918ead070a | ||
|
a461d20af9 | ||
|
8fd9345d2d | ||
|
98a2c1c638 | ||
|
bab574d674 | ||
|
57a6a64d8c | ||
|
de78992966 | ||
|
42670a46d4 | ||
|
b55d0b54ea | ||
|
418ef3f8bc | ||
|
b420bbf855 | ||
|
8111a32c72 | ||
|
721e9a7356 | ||
|
1b1db0081f | ||
|
6a7cfd13f7 | ||
|
51f3245030 | ||
|
baf6aaef1d | ||
|
9ca149a487 | ||
|
2fce90ab76 | ||
|
08fd6c8ab6 | ||
|
042194e3f6 | ||
|
120e38c72a | ||
|
38b5ea9066 | ||
|
fe5e061fff | ||
|
f6b965872f | ||
|
22c3b71196 | ||
|
1da5729f89 | ||
|
8e3ba4bd6c | ||
|
b30aabf635 | ||
|
7425d4aa1a | ||
|
cf12e3924f | ||
|
be55e1d2e1 | ||
|
c9aedee24f | ||
|
32cfdc98b2 | ||
|
b2c4b771d8 | ||
|
edf4e09fb2 | ||
|
2c0ddd10a0 | ||
|
bf4b53390b | ||
|
34d02304d3 | ||
|
29bc7cf5d5 | ||
|
a39a4b1080 | ||
|
229fffcd56 | ||
|
3c586d80c8 | ||
|
0ff90e4a5c | ||
|
5d6d378f24 | ||
|
14dad25adc | ||
|
906a181b97 | ||
|
7b09ec9ccb | ||
|
d5bea37c96 | ||
|
ad24989b31 | ||
|
d36176ad85 |
34
.github/actions/debug/action.yml
vendored
34
.github/actions/debug/action.yml
vendored
@ -4,15 +4,31 @@ description: Prints workflow debug info
|
|||||||
runs:
|
runs:
|
||||||
using: "composite"
|
using: "composite"
|
||||||
steps:
|
steps:
|
||||||
- name: Print envs
|
- name: Envs, event.json and contexts
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
echo "::group::Envs"
|
echo '::group::Environment variables'
|
||||||
env
|
env | sort
|
||||||
echo "::endgroup::"
|
echo '::endgroup::'
|
||||||
- name: Print Event.json
|
|
||||||
shell: bash
|
echo '::group::event.json'
|
||||||
run: |
|
|
||||||
echo "::group::Event.json"
|
|
||||||
python3 -m json.tool "$GITHUB_EVENT_PATH"
|
python3 -m json.tool "$GITHUB_EVENT_PATH"
|
||||||
echo "::endgroup::"
|
echo '::endgroup::'
|
||||||
|
|
||||||
|
cat << 'EOF'
|
||||||
|
::group::github context
|
||||||
|
${{ toJSON(github) }}
|
||||||
|
::endgroup::
|
||||||
|
|
||||||
|
::group::env context
|
||||||
|
${{ toJSON(env) }}
|
||||||
|
::endgroup::
|
||||||
|
|
||||||
|
::group::runner context
|
||||||
|
${{ toJSON(runner) }}
|
||||||
|
::endgroup::
|
||||||
|
|
||||||
|
::group::job context
|
||||||
|
${{ toJSON(job) }}
|
||||||
|
::endgroup::
|
||||||
|
EOF
|
||||||
|
2
.github/workflows/backport_branches.yml
vendored
2
.github/workflows/backport_branches.yml
vendored
@ -27,6 +27,8 @@ jobs:
|
|||||||
clear-repository: true # to ensure correct digests
|
clear-repository: true # to ensure correct digests
|
||||||
fetch-depth: 0 # to get version
|
fetch-depth: 0 # to get version
|
||||||
filter: tree:0
|
filter: tree:0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: Labels check
|
- name: Labels check
|
||||||
run: |
|
run: |
|
||||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||||
|
2
.github/workflows/cherry_pick.yml
vendored
2
.github/workflows/cherry_pick.yml
vendored
@ -33,6 +33,8 @@ jobs:
|
|||||||
clear-repository: true
|
clear-repository: true
|
||||||
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
|
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: Cherry pick
|
- name: Cherry pick
|
||||||
run: |
|
run: |
|
||||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||||
|
4
.github/workflows/create_release.yml
vendored
4
.github/workflows/create_release.yml
vendored
@ -56,13 +56,13 @@ jobs:
|
|||||||
GH_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
|
GH_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
|
||||||
runs-on: [self-hosted, release-maker]
|
runs-on: [self-hosted, release-maker]
|
||||||
steps:
|
steps:
|
||||||
- name: DebugInfo
|
|
||||||
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
|
|
||||||
- name: Check out repository code
|
- name: Check out repository code
|
||||||
uses: ClickHouse/checkout@v1
|
uses: ClickHouse/checkout@v1
|
||||||
with:
|
with:
|
||||||
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
|
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: Prepare Release Info
|
- name: Prepare Release Info
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
|
1
.github/workflows/docker_test_images.yml
vendored
1
.github/workflows/docker_test_images.yml
vendored
@ -11,6 +11,7 @@ name: Build docker images
|
|||||||
required: false
|
required: false
|
||||||
type: boolean
|
type: boolean
|
||||||
default: false
|
default: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
DockerBuildAarch64:
|
DockerBuildAarch64:
|
||||||
runs-on: [self-hosted, style-checker-aarch64]
|
runs-on: [self-hosted, style-checker-aarch64]
|
||||||
|
5
.github/workflows/jepsen.yml
vendored
5
.github/workflows/jepsen.yml
vendored
@ -8,20 +8,21 @@ on: # yamllint disable-line rule:truthy
|
|||||||
schedule:
|
schedule:
|
||||||
- cron: '0 */6 * * *'
|
- cron: '0 */6 * * *'
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
RunConfig:
|
RunConfig:
|
||||||
runs-on: [self-hosted, style-checker-aarch64]
|
runs-on: [self-hosted, style-checker-aarch64]
|
||||||
outputs:
|
outputs:
|
||||||
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
||||||
steps:
|
steps:
|
||||||
- name: DebugInfo
|
|
||||||
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
|
|
||||||
- name: Check out repository code
|
- name: Check out repository code
|
||||||
uses: ClickHouse/checkout@v1
|
uses: ClickHouse/checkout@v1
|
||||||
with:
|
with:
|
||||||
clear-repository: true # to ensure correct digests
|
clear-repository: true # to ensure correct digests
|
||||||
fetch-depth: 0 # to get version
|
fetch-depth: 0 # to get version
|
||||||
filter: tree:0
|
filter: tree:0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: PrepareRunConfig
|
- name: PrepareRunConfig
|
||||||
id: runconfig
|
id: runconfig
|
||||||
run: |
|
run: |
|
||||||
|
4
.github/workflows/master.yml
vendored
4
.github/workflows/master.yml
vendored
@ -15,14 +15,14 @@ jobs:
|
|||||||
outputs:
|
outputs:
|
||||||
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
||||||
steps:
|
steps:
|
||||||
- name: DebugInfo
|
|
||||||
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
|
|
||||||
- name: Check out repository code
|
- name: Check out repository code
|
||||||
uses: ClickHouse/checkout@v1
|
uses: ClickHouse/checkout@v1
|
||||||
with:
|
with:
|
||||||
clear-repository: true # to ensure correct digests
|
clear-repository: true # to ensure correct digests
|
||||||
fetch-depth: 0 # to get version
|
fetch-depth: 0 # to get version
|
||||||
filter: tree:0
|
filter: tree:0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: Merge sync PR
|
- name: Merge sync PR
|
||||||
run: |
|
run: |
|
||||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||||
|
4
.github/workflows/merge_queue.yml
vendored
4
.github/workflows/merge_queue.yml
vendored
@ -14,14 +14,14 @@ jobs:
|
|||||||
outputs:
|
outputs:
|
||||||
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
||||||
steps:
|
steps:
|
||||||
- name: DebugInfo
|
|
||||||
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
|
|
||||||
- name: Check out repository code
|
- name: Check out repository code
|
||||||
uses: ClickHouse/checkout@v1
|
uses: ClickHouse/checkout@v1
|
||||||
with:
|
with:
|
||||||
clear-repository: true # to ensure correct digests
|
clear-repository: true # to ensure correct digests
|
||||||
fetch-depth: 0 # to get a version
|
fetch-depth: 0 # to get a version
|
||||||
filter: tree:0
|
filter: tree:0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: Cancel PR workflow
|
- name: Cancel PR workflow
|
||||||
run: |
|
run: |
|
||||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
|
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
|
||||||
|
4
.github/workflows/nightly.yml
vendored
4
.github/workflows/nightly.yml
vendored
@ -15,14 +15,14 @@ jobs:
|
|||||||
outputs:
|
outputs:
|
||||||
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
||||||
steps:
|
steps:
|
||||||
- name: DebugInfo
|
|
||||||
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
|
|
||||||
- name: Check out repository code
|
- name: Check out repository code
|
||||||
uses: ClickHouse/checkout@v1
|
uses: ClickHouse/checkout@v1
|
||||||
with:
|
with:
|
||||||
clear-repository: true # to ensure correct digests
|
clear-repository: true # to ensure correct digests
|
||||||
fetch-depth: 0 # to get version
|
fetch-depth: 0 # to get version
|
||||||
filter: tree:0
|
filter: tree:0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: PrepareRunConfig
|
- name: PrepareRunConfig
|
||||||
id: runconfig
|
id: runconfig
|
||||||
run: |
|
run: |
|
||||||
|
4
.github/workflows/pull_request.yml
vendored
4
.github/workflows/pull_request.yml
vendored
@ -25,14 +25,14 @@ jobs:
|
|||||||
outputs:
|
outputs:
|
||||||
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
data: ${{ steps.runconfig.outputs.CI_DATA }}
|
||||||
steps:
|
steps:
|
||||||
- name: DebugInfo
|
|
||||||
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
|
|
||||||
- name: Check out repository code
|
- name: Check out repository code
|
||||||
uses: ClickHouse/checkout@v1
|
uses: ClickHouse/checkout@v1
|
||||||
with:
|
with:
|
||||||
clear-repository: true # to ensure correct digests
|
clear-repository: true # to ensure correct digests
|
||||||
fetch-depth: 0 # to get a version
|
fetch-depth: 0 # to get a version
|
||||||
filter: tree:0
|
filter: tree:0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: Cancel previous Sync PR workflow
|
- name: Cancel previous Sync PR workflow
|
||||||
run: |
|
run: |
|
||||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
|
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
|
||||||
|
2
.github/workflows/release_branches.yml
vendored
2
.github/workflows/release_branches.yml
vendored
@ -24,6 +24,8 @@ jobs:
|
|||||||
clear-repository: true # to ensure correct digests
|
clear-repository: true # to ensure correct digests
|
||||||
fetch-depth: 0 # to get version
|
fetch-depth: 0 # to get version
|
||||||
filter: tree:0
|
filter: tree:0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: Labels check
|
- name: Labels check
|
||||||
run: |
|
run: |
|
||||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||||
|
4
.github/workflows/reusable_simple_job.yml
vendored
4
.github/workflows/reusable_simple_job.yml
vendored
@ -62,8 +62,6 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}
|
GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}
|
||||||
steps:
|
steps:
|
||||||
- name: DebugInfo
|
|
||||||
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
|
|
||||||
- name: Check out repository code
|
- name: Check out repository code
|
||||||
uses: ClickHouse/checkout@v1
|
uses: ClickHouse/checkout@v1
|
||||||
with:
|
with:
|
||||||
@ -72,6 +70,8 @@ jobs:
|
|||||||
submodules: ${{inputs.submodules}}
|
submodules: ${{inputs.submodules}}
|
||||||
fetch-depth: ${{inputs.checkout_depth}}
|
fetch-depth: ${{inputs.checkout_depth}}
|
||||||
filter: tree:0
|
filter: tree:0
|
||||||
|
- name: Debug Info
|
||||||
|
uses: ./.github/actions/debug
|
||||||
- name: Set build envs
|
- name: Set build envs
|
||||||
run: |
|
run: |
|
||||||
cat >> "$GITHUB_ENV" << 'EOF'
|
cat >> "$GITHUB_ENV" << 'EOF'
|
||||||
|
@ -107,6 +107,10 @@ The vector similarity index currently does not work with per-table, non-default
|
|||||||
[here](https://github.com/ClickHouse/ClickHouse/pull/51325#issuecomment-1605920475)). If necessary, the value must be changed in config.xml.
|
[here](https://github.com/ClickHouse/ClickHouse/pull/51325#issuecomment-1605920475)). If necessary, the value must be changed in config.xml.
|
||||||
:::
|
:::
|
||||||
|
|
||||||
|
Vector index creation is known to be slow. To speed the process up, index creation can be parallelized. The maximum number of threads can be
|
||||||
|
configured using server configuration
|
||||||
|
setting [max_build_vector_similarity_index_thread_pool_size](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters_max_build_vector_similarity_index_thread_pool_size).
|
||||||
|
|
||||||
ANN indexes are built during column insertion and merge. As a result, `INSERT` and `OPTIMIZE` statements will be slower than for ordinary
|
ANN indexes are built during column insertion and merge. As a result, `INSERT` and `OPTIMIZE` statements will be slower than for ordinary
|
||||||
tables. ANNIndexes are ideally used only with immutable or rarely changed data, respectively when are far more read requests than write
|
tables. ANNIndexes are ideally used only with immutable or rarely changed data, respectively when are far more read requests than write
|
||||||
requests.
|
requests.
|
||||||
|
@ -1396,6 +1396,7 @@ SELECT * FROM json_each_row_nested
|
|||||||
- [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - ignore unknown keys in json object for named tuples. Default value - `false`.
|
- [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - ignore unknown keys in json object for named tuples. Default value - `false`.
|
||||||
- [input_format_json_compact_allow_variable_number_of_columns](/docs/en/operations/settings/settings-formats.md/#input_format_json_compact_allow_variable_number_of_columns) - allow variable number of columns in JSONCompact/JSONCompactEachRow format, ignore extra columns and use default values on missing columns. Default value - `false`.
|
- [input_format_json_compact_allow_variable_number_of_columns](/docs/en/operations/settings/settings-formats.md/#input_format_json_compact_allow_variable_number_of_columns) - allow variable number of columns in JSONCompact/JSONCompactEachRow format, ignore extra columns and use default values on missing columns. Default value - `false`.
|
||||||
- [input_format_json_throw_on_bad_escape_sequence](/docs/en/operations/settings/settings-formats.md/#input_format_json_throw_on_bad_escape_sequence) - throw an exception if JSON string contains bad escape sequence. If disabled, bad escape sequences will remain as is in the data. Default value - `true`.
|
- [input_format_json_throw_on_bad_escape_sequence](/docs/en/operations/settings/settings-formats.md/#input_format_json_throw_on_bad_escape_sequence) - throw an exception if JSON string contains bad escape sequence. If disabled, bad escape sequences will remain as is in the data. Default value - `true`.
|
||||||
|
- [input_format_json_empty_as_default](/docs/en/operations/settings/settings-formats.md/#input_format_json_empty_as_default) - treat empty fields in JSON input as default values. Default value - `false`. For complex default expressions [input_format_defaults_for_omitted_fields](/docs/en/operations/settings/settings-formats.md/#input_format_defaults_for_omitted_fields) must be enabled too.
|
||||||
- [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`.
|
- [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`.
|
||||||
- [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`.
|
- [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`.
|
||||||
- [output_format_json_quote_denormals](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`.
|
- [output_format_json_quote_denormals](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`.
|
||||||
|
@ -491,6 +491,14 @@ Type: Double
|
|||||||
|
|
||||||
Default: 0.9
|
Default: 0.9
|
||||||
|
|
||||||
|
## max_build_vector_similarity_index_thread_pool_size {#server_configuration_parameters_max_build_vector_similarity_index_thread_pool_size}
|
||||||
|
|
||||||
|
The maximum number of threads to use for building vector indexes. 0 means all cores.
|
||||||
|
|
||||||
|
Type: UInt64
|
||||||
|
|
||||||
|
Default: 16
|
||||||
|
|
||||||
## cgroups_memory_usage_observer_wait_time
|
## cgroups_memory_usage_observer_wait_time
|
||||||
|
|
||||||
Interval in seconds during which the server's maximum allowed memory consumption is adjusted by the corresponding threshold in cgroups. (see
|
Interval in seconds during which the server's maximum allowed memory consumption is adjusted by the corresponding threshold in cgroups. (see
|
||||||
|
@ -752,6 +752,17 @@ Possible values:
|
|||||||
|
|
||||||
Default value: 0.
|
Default value: 0.
|
||||||
|
|
||||||
|
### input_format_json_empty_as_default {#input_format_json_empty_as_default}
|
||||||
|
|
||||||
|
When enabled, replace empty input fields in JSON with default values. For complex default expressions `input_format_defaults_for_omitted_fields` must be enabled too.
|
||||||
|
|
||||||
|
Possible values:
|
||||||
|
|
||||||
|
+ 0 — Disable.
|
||||||
|
+ 1 — Enable.
|
||||||
|
|
||||||
|
Default value: 0.
|
||||||
|
|
||||||
## TSV format settings {#tsv-format-settings}
|
## TSV format settings {#tsv-format-settings}
|
||||||
|
|
||||||
### input_format_tsv_empty_as_default {#input_format_tsv_empty_as_default}
|
### input_format_tsv_empty_as_default {#input_format_tsv_empty_as_default}
|
||||||
|
@ -178,6 +178,9 @@
|
|||||||
M(ObjectStorageAzureThreads, "Number of threads in the AzureObjectStorage thread pool.") \
|
M(ObjectStorageAzureThreads, "Number of threads in the AzureObjectStorage thread pool.") \
|
||||||
M(ObjectStorageAzureThreadsActive, "Number of threads in the AzureObjectStorage thread pool running a task.") \
|
M(ObjectStorageAzureThreadsActive, "Number of threads in the AzureObjectStorage thread pool running a task.") \
|
||||||
M(ObjectStorageAzureThreadsScheduled, "Number of queued or active jobs in the AzureObjectStorage thread pool.") \
|
M(ObjectStorageAzureThreadsScheduled, "Number of queued or active jobs in the AzureObjectStorage thread pool.") \
|
||||||
|
M(BuildVectorSimilarityIndexThreads, "Number of threads in the build vector similarity index thread pool.") \
|
||||||
|
M(BuildVectorSimilarityIndexThreadsActive, "Number of threads in the build vector similarity index thread pool running a task.") \
|
||||||
|
M(BuildVectorSimilarityIndexThreadsScheduled, "Number of queued or active jobs in the build vector similarity index thread pool.") \
|
||||||
\
|
\
|
||||||
M(DiskPlainRewritableAzureDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for AzureObjectStorage.") \
|
M(DiskPlainRewritableAzureDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for AzureObjectStorage.") \
|
||||||
M(DiskPlainRewritableLocalDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for LocalObjectStorage.") \
|
M(DiskPlainRewritableLocalDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for LocalObjectStorage.") \
|
||||||
|
@ -609,6 +609,7 @@
|
|||||||
M(728, UNEXPECTED_DATA_TYPE) \
|
M(728, UNEXPECTED_DATA_TYPE) \
|
||||||
M(729, ILLEGAL_TIME_SERIES_TAGS) \
|
M(729, ILLEGAL_TIME_SERIES_TAGS) \
|
||||||
M(730, REFRESH_FAILED) \
|
M(730, REFRESH_FAILED) \
|
||||||
|
M(731, QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE) \
|
||||||
\
|
\
|
||||||
M(900, DISTRIBUTED_CACHE_ERROR) \
|
M(900, DISTRIBUTED_CACHE_ERROR) \
|
||||||
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \
|
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \
|
||||||
|
@ -63,6 +63,7 @@ static struct InitFiu
|
|||||||
REGULAR(keepermap_fail_drop_data) \
|
REGULAR(keepermap_fail_drop_data) \
|
||||||
REGULAR(lazy_pipe_fds_fail_close) \
|
REGULAR(lazy_pipe_fds_fail_close) \
|
||||||
PAUSEABLE(infinite_sleep) \
|
PAUSEABLE(infinite_sleep) \
|
||||||
|
PAUSEABLE(stop_moving_part_before_swap_with_active) \
|
||||||
|
|
||||||
|
|
||||||
namespace FailPoints
|
namespace FailPoints
|
||||||
|
@ -50,7 +50,7 @@ namespace DB
|
|||||||
M(UInt32, asynchronous_heavy_metrics_update_period_s, 120, "Period in seconds for updating heavy asynchronous metrics.", 0) \
|
M(UInt32, asynchronous_heavy_metrics_update_period_s, 120, "Period in seconds for updating heavy asynchronous metrics.", 0) \
|
||||||
M(String, default_database, "default", "Default database name.", 0) \
|
M(String, default_database, "default", "Default database name.", 0) \
|
||||||
M(String, tmp_policy, "", "Policy for storage with temporary data.", 0) \
|
M(String, tmp_policy, "", "Policy for storage with temporary data.", 0) \
|
||||||
M(UInt64, max_temporary_data_on_disk_size, 0, "The maximum amount of storage that could be used for external aggregation, joins or sorting., ", 0) \
|
M(UInt64, max_temporary_data_on_disk_size, 0, "The maximum amount of storage that could be used for external aggregation, joins or sorting.", 0) \
|
||||||
M(String, temporary_data_in_cache, "", "Cache disk name for temporary data.", 0) \
|
M(String, temporary_data_in_cache, "", "Cache disk name for temporary data.", 0) \
|
||||||
M(UInt64, aggregate_function_group_array_max_element_size, 0xFFFFFF, "Max array element size in bytes for groupArray function. This limit is checked at serialization and help to avoid large state size.", 0) \
|
M(UInt64, aggregate_function_group_array_max_element_size, 0xFFFFFF, "Max array element size in bytes for groupArray function. This limit is checked at serialization and help to avoid large state size.", 0) \
|
||||||
M(GroupArrayActionWhenLimitReached, aggregate_function_group_array_action_when_limit_is_reached, GroupArrayActionWhenLimitReached::THROW, "Action to execute when max array element size is exceeded in groupArray: `throw` exception, or `discard` extra values", 0) \
|
M(GroupArrayActionWhenLimitReached, aggregate_function_group_array_action_when_limit_is_reached, GroupArrayActionWhenLimitReached::THROW, "Action to execute when max array element size is exceeded in groupArray: `throw` exception, or `discard` extra values", 0) \
|
||||||
@ -65,6 +65,7 @@ namespace DB
|
|||||||
M(UInt64, async_insert_threads, 16, "Maximum number of threads to actually parse and insert data in background. Zero means asynchronous mode is disabled", 0) \
|
M(UInt64, async_insert_threads, 16, "Maximum number of threads to actually parse and insert data in background. Zero means asynchronous mode is disabled", 0) \
|
||||||
M(Bool, async_insert_queue_flush_on_shutdown, true, "If true queue of asynchronous inserts is flushed on graceful shutdown", 0) \
|
M(Bool, async_insert_queue_flush_on_shutdown, true, "If true queue of asynchronous inserts is flushed on graceful shutdown", 0) \
|
||||||
M(Bool, ignore_empty_sql_security_in_create_view_query, true, "If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. This setting is only necessary for the migration period and will become obsolete in 24.4", 0) \
|
M(Bool, ignore_empty_sql_security_in_create_view_query, true, "If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. This setting is only necessary for the migration period and will become obsolete in 24.4", 0) \
|
||||||
|
M(UInt64, max_build_vector_similarity_index_thread_pool_size, 16, "The maximum number of threads to use to build vector similarity indexes. 0 means all cores.", 0) \
|
||||||
\
|
\
|
||||||
/* Database Catalog */ \
|
/* Database Catalog */ \
|
||||||
M(UInt64, database_atomic_delay_before_drop_table_sec, 8 * 60, "The delay during which a dropped table can be restored using the UNDROP statement. If DROP TABLE ran with a SYNC modifier, the setting is ignored.", 0) \
|
M(UInt64, database_atomic_delay_before_drop_table_sec, 8 * 60, "The delay during which a dropped table can be restored using the UNDROP statement. If DROP TABLE ran with a SYNC modifier, the setting is ignored.", 0) \
|
||||||
|
@ -1144,6 +1144,7 @@ class IColumn;
|
|||||||
M(Bool, input_format_try_infer_variants, false, "Try to infer the Variant type in text formats when there is more than one possible type for column/array elements", 0) \
|
M(Bool, input_format_try_infer_variants, false, "Try to infer the Variant type in text formats when there is more than one possible type for column/array elements", 0) \
|
||||||
M(Bool, type_json_skip_duplicated_paths, false, "When enabled, during parsing JSON object into JSON type duplicated paths will be ignored and only the first one will be inserted instead of an exception", 0) \
|
M(Bool, type_json_skip_duplicated_paths, false, "When enabled, during parsing JSON object into JSON type duplicated paths will be ignored and only the first one will be inserted instead of an exception", 0) \
|
||||||
M(UInt64, input_format_json_max_depth, 1000, "Maximum depth of a field in JSON. This is not a strict limit, it does not have to be applied precisely.", 0) \
|
M(UInt64, input_format_json_max_depth, 1000, "Maximum depth of a field in JSON. This is not a strict limit, it does not have to be applied precisely.", 0) \
|
||||||
|
M(Bool, input_format_json_empty_as_default, false, "Treat empty fields in JSON input as default values.", 0) \
|
||||||
M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
|
M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
|
||||||
M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
|
M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
|
||||||
M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
|
M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
|
||||||
|
@ -71,6 +71,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
|
|||||||
},
|
},
|
||||||
{"24.9",
|
{"24.9",
|
||||||
{
|
{
|
||||||
|
{"input_format_json_empty_as_default", false, false, "Added new setting to allow to treat empty fields in JSON input as default values."},
|
||||||
{"input_format_try_infer_variants", false, false, "Try to infer Variant type in text formats when there is more than one possible type for column/array elements"},
|
{"input_format_try_infer_variants", false, false, "Try to infer Variant type in text formats when there is more than one possible type for column/array elements"},
|
||||||
{"join_output_by_rowlist_perkey_rows_threshold", 0, 5, "The lower limit of per-key average rows in the right table to determine whether to output by row list in hash join."},
|
{"join_output_by_rowlist_perkey_rows_threshold", 0, 5, "The lower limit of per-key average rows in the right table to determine whether to output by row list in hash join."},
|
||||||
{"create_if_not_exists", false, false, "New setting."},
|
{"create_if_not_exists", false, false, "New setting."},
|
||||||
|
@ -11,6 +11,7 @@
|
|||||||
#include <IO/WriteBufferFromString.h>
|
#include <IO/WriteBufferFromString.h>
|
||||||
|
|
||||||
#include <Formats/FormatSettings.h>
|
#include <Formats/FormatSettings.h>
|
||||||
|
#include <Formats/JSONUtils.h>
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
@ -615,28 +616,49 @@ void SerializationArray::serializeTextJSONPretty(const IColumn & column, size_t
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void SerializationArray::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
template <typename ReturnType>
|
||||||
|
ReturnType SerializationArray::deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
{
|
{
|
||||||
deserializeTextImpl(column, istr,
|
auto deserialize_nested = [&settings, this](IColumn & nested_column, ReadBuffer & buf) -> ReturnType
|
||||||
[&](IColumn & nested_column)
|
{
|
||||||
|
if constexpr (std::is_same_v<ReturnType, void>)
|
||||||
{
|
{
|
||||||
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column))
|
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column))
|
||||||
SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(nested_column, istr, settings, nested);
|
SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(nested_column, buf, settings, nested);
|
||||||
else
|
else
|
||||||
nested->deserializeTextJSON(nested_column, istr, settings);
|
nested->deserializeTextJSON(nested_column, buf, settings);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column))
|
||||||
|
return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(nested_column, buf, settings, nested);
|
||||||
|
return nested->tryDeserializeTextJSON(nested_column, buf, settings);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (settings.json.empty_as_default)
|
||||||
|
return deserializeTextImpl<ReturnType>(column, istr,
|
||||||
|
[&deserialize_nested, &istr](IColumn & nested_column) -> ReturnType
|
||||||
|
{
|
||||||
|
return JSONUtils::deserializeEmpyStringAsDefaultOrNested<ReturnType>(nested_column, istr, deserialize_nested);
|
||||||
}, false);
|
}, false);
|
||||||
|
else
|
||||||
|
return deserializeTextImpl<ReturnType>(column, istr,
|
||||||
|
[&deserialize_nested, &istr](IColumn & nested_column) -> ReturnType
|
||||||
|
{
|
||||||
|
return deserialize_nested(nested_column, istr);
|
||||||
|
}, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void SerializationArray::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
|
{
|
||||||
|
deserializeTextJSONImpl<void>(column, istr, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool SerializationArray::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
bool SerializationArray::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
{
|
{
|
||||||
auto read_nested = [&](IColumn & nested_column)
|
return deserializeTextJSONImpl<bool>(column, istr, settings);
|
||||||
{
|
|
||||||
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column))
|
|
||||||
return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(nested_column, istr, settings, nested);
|
|
||||||
return nested->tryDeserializeTextJSON(nested_column, istr, settings);
|
|
||||||
};
|
|
||||||
|
|
||||||
return deserializeTextImpl<bool>(column, istr, std::move(read_nested), false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -82,6 +82,10 @@ public:
|
|||||||
SerializationPtr create(const SerializationPtr & prev) const override;
|
SerializationPtr create(const SerializationPtr & prev) const override;
|
||||||
ColumnPtr create(const ColumnPtr & prev) const override;
|
ColumnPtr create(const ColumnPtr & prev) const override;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
private:
|
||||||
|
template <typename ReturnType>
|
||||||
|
ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -6,6 +6,7 @@
|
|||||||
#include <Columns/ColumnMap.h>
|
#include <Columns/ColumnMap.h>
|
||||||
#include <Core/Field.h>
|
#include <Core/Field.h>
|
||||||
#include <Formats/FormatSettings.h>
|
#include <Formats/FormatSettings.h>
|
||||||
|
#include <Formats/JSONUtils.h>
|
||||||
#include <Common/assert_cast.h>
|
#include <Common/assert_cast.h>
|
||||||
#include <Common/quoteString.h>
|
#include <Common/quoteString.h>
|
||||||
#include <IO/WriteHelpers.h>
|
#include <IO/WriteHelpers.h>
|
||||||
@ -316,28 +317,52 @@ void SerializationMap::serializeTextJSONPretty(const IColumn & column, size_t ro
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void SerializationMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
template <typename ReturnType>
|
||||||
|
ReturnType SerializationMap::deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
{
|
{
|
||||||
deserializeTextImpl(column, istr,
|
auto deserialize_nested = [&settings](IColumn & subcolumn, ReadBuffer & buf, const SerializationPtr & subcolumn_serialization) -> ReturnType
|
||||||
[&settings](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn)
|
{
|
||||||
|
if constexpr (std::is_same_v<ReturnType, void>)
|
||||||
{
|
{
|
||||||
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(subcolumn))
|
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(subcolumn))
|
||||||
SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(subcolumn, buf, settings, subcolumn_serialization);
|
SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(subcolumn, buf, settings, subcolumn_serialization);
|
||||||
else
|
else
|
||||||
subcolumn_serialization->deserializeTextJSON(subcolumn, buf, settings);
|
subcolumn_serialization->deserializeTextJSON(subcolumn, buf, settings);
|
||||||
});
|
}
|
||||||
}
|
else
|
||||||
|
|
||||||
bool SerializationMap::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
|
||||||
{
|
|
||||||
auto reader = [&settings](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn)
|
|
||||||
{
|
{
|
||||||
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(subcolumn))
|
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(subcolumn))
|
||||||
return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(subcolumn, buf, settings, subcolumn_serialization);
|
return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(subcolumn, buf, settings, subcolumn_serialization);
|
||||||
return subcolumn_serialization->tryDeserializeTextJSON(subcolumn, buf, settings);
|
return subcolumn_serialization->tryDeserializeTextJSON(subcolumn, buf, settings);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
return deserializeTextImpl<bool>(column, istr, reader);
|
if (settings.json.empty_as_default)
|
||||||
|
return deserializeTextImpl<ReturnType>(column, istr,
|
||||||
|
[&deserialize_nested](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) -> ReturnType
|
||||||
|
{
|
||||||
|
return JSONUtils::deserializeEmpyStringAsDefaultOrNested<ReturnType>(subcolumn, buf,
|
||||||
|
[&deserialize_nested, &subcolumn_serialization](IColumn & subcolumn_, ReadBuffer & buf_) -> ReturnType
|
||||||
|
{
|
||||||
|
return deserialize_nested(subcolumn_, buf_, subcolumn_serialization);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
else
|
||||||
|
return deserializeTextImpl<ReturnType>(column, istr,
|
||||||
|
[&deserialize_nested](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) -> ReturnType
|
||||||
|
{
|
||||||
|
return deserialize_nested(subcolumn, buf, subcolumn_serialization);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
void SerializationMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
|
{
|
||||||
|
deserializeTextJSONImpl<void>(column, istr, settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool SerializationMap::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
|
{
|
||||||
|
return deserializeTextJSONImpl<bool>(column, istr, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
void SerializationMap::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
void SerializationMap::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||||
|
@ -74,6 +74,9 @@ private:
|
|||||||
|
|
||||||
template <typename ReturnType = void, typename Reader>
|
template <typename ReturnType = void, typename Reader>
|
||||||
ReturnType deserializeTextImpl(IColumn & column, ReadBuffer & istr, Reader && reader) const;
|
ReturnType deserializeTextImpl(IColumn & column, ReadBuffer & istr, Reader && reader) const;
|
||||||
|
|
||||||
|
template <typename ReturnType>
|
||||||
|
ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -5,6 +5,7 @@
|
|||||||
#include <Core/Field.h>
|
#include <Core/Field.h>
|
||||||
#include <Columns/ColumnTuple.h>
|
#include <Columns/ColumnTuple.h>
|
||||||
#include <Common/assert_cast.h>
|
#include <Common/assert_cast.h>
|
||||||
|
#include <Formats/JSONUtils.h>
|
||||||
#include <IO/WriteHelpers.h>
|
#include <IO/WriteHelpers.h>
|
||||||
#include <IO/ReadHelpers.h>
|
#include <IO/ReadHelpers.h>
|
||||||
#include <IO/ReadBufferFromString.h>
|
#include <IO/ReadBufferFromString.h>
|
||||||
@ -313,27 +314,9 @@ void SerializationTuple::serializeTextJSONPretty(const IColumn & column, size_t
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <typename ReturnType>
|
template <typename ReturnType>
|
||||||
ReturnType SerializationTuple::deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
ReturnType SerializationTuple::deserializeTupleJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, auto && deserialize_element) const
|
||||||
{
|
{
|
||||||
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
|
static constexpr auto throw_exception = std::is_same_v<ReturnType, void>;
|
||||||
|
|
||||||
auto deserialize_element = [&](IColumn & element_column, size_t element_pos)
|
|
||||||
{
|
|
||||||
if constexpr (throw_exception)
|
|
||||||
{
|
|
||||||
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column))
|
|
||||||
SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(element_column, istr, settings, elems[element_pos]);
|
|
||||||
else
|
|
||||||
elems[element_pos]->deserializeTextJSON(element_column, istr, settings);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column))
|
|
||||||
return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(element_column, istr, settings, elems[element_pos]);
|
|
||||||
return elems[element_pos]->tryDeserializeTextJSON(element_column, istr, settings);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if (settings.json.read_named_tuples_as_objects
|
if (settings.json.read_named_tuples_as_objects
|
||||||
&& have_explicit_names)
|
&& have_explicit_names)
|
||||||
@ -506,12 +489,51 @@ ReturnType SerializationTuple::deserializeTextJSONImpl(IColumn & column, ReadBuf
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void SerializationTuple::deserializeTextJSON(DB::IColumn & column, DB::ReadBuffer & istr, const DB::FormatSettings & settings) const
|
template <typename ReturnType>
|
||||||
|
ReturnType SerializationTuple::deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
{
|
{
|
||||||
deserializeTextJSONImpl(column, istr, settings);
|
auto deserialize_nested = [&settings](IColumn & nested_column, ReadBuffer & buf, const SerializationPtr & nested_column_serialization) -> ReturnType
|
||||||
|
{
|
||||||
|
if constexpr (std::is_same_v<ReturnType, void>)
|
||||||
|
{
|
||||||
|
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column))
|
||||||
|
SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(nested_column, buf, settings, nested_column_serialization);
|
||||||
|
else
|
||||||
|
nested_column_serialization->deserializeTextJSON(nested_column, buf, settings);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column))
|
||||||
|
return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(nested_column, buf, settings, nested_column_serialization);
|
||||||
|
else
|
||||||
|
return nested_column_serialization->tryDeserializeTextJSON(nested_column, buf, settings);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (settings.json.empty_as_default)
|
||||||
|
return deserializeTupleJSONImpl<ReturnType>(column, istr, settings,
|
||||||
|
[&deserialize_nested, &istr, this](IColumn & nested_column, size_t element_pos) -> ReturnType
|
||||||
|
{
|
||||||
|
return JSONUtils::deserializeEmpyStringAsDefaultOrNested<ReturnType>(nested_column, istr,
|
||||||
|
[&deserialize_nested, element_pos, this](IColumn & nested_column_, ReadBuffer & buf) -> ReturnType
|
||||||
|
{
|
||||||
|
return deserialize_nested(nested_column_, buf, elems[element_pos]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
else
|
||||||
|
return deserializeTupleJSONImpl<ReturnType>(column, istr, settings,
|
||||||
|
[&deserialize_nested, &istr, this](IColumn & nested_column, size_t element_pos) -> ReturnType
|
||||||
|
{
|
||||||
|
return deserialize_nested(nested_column, istr, elems[element_pos]);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
bool SerializationTuple::tryDeserializeTextJSON(DB::IColumn & column, DB::ReadBuffer & istr, const DB::FormatSettings & settings) const
|
void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
|
{
|
||||||
|
deserializeTextJSONImpl<void>(column, istr, settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool SerializationTuple::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
{
|
{
|
||||||
return deserializeTextJSONImpl<bool>(column, istr, settings);
|
return deserializeTextJSONImpl<bool>(column, istr, settings);
|
||||||
}
|
}
|
||||||
|
@ -81,7 +81,10 @@ private:
|
|||||||
template <typename ReturnType = void>
|
template <typename ReturnType = void>
|
||||||
ReturnType deserializeTextImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const;
|
ReturnType deserializeTextImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const;
|
||||||
|
|
||||||
template <typename ReturnType = void>
|
template <typename ReturnType>
|
||||||
|
ReturnType deserializeTupleJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, auto && deserialize_element) const;
|
||||||
|
|
||||||
|
template <typename ReturnType>
|
||||||
ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const;
|
ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const;
|
||||||
|
|
||||||
template <typename ReturnType = void>
|
template <typename ReturnType = void>
|
||||||
|
@ -152,6 +152,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
|
|||||||
format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects;
|
format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects;
|
||||||
format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence;
|
format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence;
|
||||||
format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields;
|
format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields;
|
||||||
|
format_settings.json.empty_as_default = settings.input_format_json_empty_as_default;
|
||||||
format_settings.json.type_json_skip_duplicated_paths = settings.type_json_skip_duplicated_paths;
|
format_settings.json.type_json_skip_duplicated_paths = settings.type_json_skip_duplicated_paths;
|
||||||
format_settings.null_as_default = settings.input_format_null_as_default;
|
format_settings.null_as_default = settings.input_format_null_as_default;
|
||||||
format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields;
|
format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields;
|
||||||
|
@ -237,6 +237,7 @@ struct FormatSettings
|
|||||||
bool infer_incomplete_types_as_strings = true;
|
bool infer_incomplete_types_as_strings = true;
|
||||||
bool throw_on_bad_escape_sequence = true;
|
bool throw_on_bad_escape_sequence = true;
|
||||||
bool ignore_unnecessary_fields = true;
|
bool ignore_unnecessary_fields = true;
|
||||||
|
bool empty_as_default = false;
|
||||||
bool type_json_skip_duplicated_paths = false;
|
bool type_json_skip_duplicated_paths = false;
|
||||||
} json{};
|
} json{};
|
||||||
|
|
||||||
|
@ -2,12 +2,14 @@
|
|||||||
#include <Formats/JSONUtils.h>
|
#include <Formats/JSONUtils.h>
|
||||||
#include <Formats/ReadSchemaUtils.h>
|
#include <Formats/ReadSchemaUtils.h>
|
||||||
#include <Formats/EscapingRuleUtils.h>
|
#include <Formats/EscapingRuleUtils.h>
|
||||||
|
#include <IO/PeekableReadBuffer.h>
|
||||||
#include <IO/ReadBufferFromString.h>
|
#include <IO/ReadBufferFromString.h>
|
||||||
#include <IO/WriteBufferValidUTF8.h>
|
#include <IO/WriteBufferValidUTF8.h>
|
||||||
#include <DataTypes/Serializations/SerializationNullable.h>
|
#include <DataTypes/Serializations/SerializationNullable.h>
|
||||||
#include <DataTypes/DataTypeNullable.h>
|
#include <DataTypes/DataTypeNullable.h>
|
||||||
#include <DataTypes/DataTypeObjectDeprecated.h>
|
#include <DataTypes/DataTypeObjectDeprecated.h>
|
||||||
#include <DataTypes/DataTypeFactory.h>
|
#include <DataTypes/DataTypeFactory.h>
|
||||||
|
#include <Common/assert_cast.h>
|
||||||
|
|
||||||
#include <base/find_symbols.h>
|
#include <base/find_symbols.h>
|
||||||
|
|
||||||
@ -286,11 +288,19 @@ namespace JSONUtils
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
auto deserialize = [as_nullable, &format_settings, &serialization](IColumn & column_, ReadBuffer & buf) -> bool
|
||||||
|
{
|
||||||
if (as_nullable)
|
if (as_nullable)
|
||||||
return SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(column, in, format_settings, serialization);
|
return SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(column_, buf, format_settings, serialization);
|
||||||
|
|
||||||
serialization->deserializeTextJSON(column, in, format_settings);
|
serialization->deserializeTextJSON(column_, buf, format_settings);
|
||||||
return true;
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (format_settings.json.empty_as_default)
|
||||||
|
return JSONUtils::deserializeEmpyStringAsDefaultOrNested<bool, false>(column, in, deserialize);
|
||||||
|
else
|
||||||
|
return deserialize(column, in);
|
||||||
}
|
}
|
||||||
catch (Exception & e)
|
catch (Exception & e)
|
||||||
{
|
{
|
||||||
@ -920,6 +930,78 @@ namespace JSONUtils
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <typename ReturnType, bool default_column_return_value>
|
||||||
|
ReturnType deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize<ReturnType> & deserialize_nested)
|
||||||
|
{
|
||||||
|
static constexpr auto throw_exception = std::is_same_v<ReturnType, void>;
|
||||||
|
|
||||||
|
static constexpr auto EMPTY_STRING = "\"\"";
|
||||||
|
static constexpr auto EMPTY_STRING_LENGTH = std::string_view(EMPTY_STRING).length();
|
||||||
|
|
||||||
|
if (istr.eof() || *istr.position() != EMPTY_STRING[0])
|
||||||
|
return deserialize_nested(column, istr);
|
||||||
|
|
||||||
|
auto do_deserialize = [](IColumn & column_, ReadBuffer & buf, auto && check_for_empty_string, auto && deserialize) -> ReturnType
|
||||||
|
{
|
||||||
|
if (check_for_empty_string(buf))
|
||||||
|
{
|
||||||
|
column_.insertDefault();
|
||||||
|
return ReturnType(default_column_return_value);
|
||||||
|
}
|
||||||
|
return deserialize(column_, buf);
|
||||||
|
};
|
||||||
|
|
||||||
|
if (istr.available() >= EMPTY_STRING_LENGTH)
|
||||||
|
{
|
||||||
|
/// We have enough data in buffer to check if we have an empty string.
|
||||||
|
auto check_for_empty_string = [](ReadBuffer & buf) -> bool
|
||||||
|
{
|
||||||
|
auto * pos = buf.position();
|
||||||
|
if (checkString(EMPTY_STRING, buf))
|
||||||
|
return true;
|
||||||
|
buf.position() = pos;
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
return do_deserialize(column, istr, check_for_empty_string, deserialize_nested);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// We don't have enough data in buffer to check if we have an empty string.
|
||||||
|
/// Use PeekableReadBuffer to make a checkpoint before checking for an
|
||||||
|
/// empty string and rollback if check was failed.
|
||||||
|
|
||||||
|
auto check_for_empty_string = [](ReadBuffer & buf) -> bool
|
||||||
|
{
|
||||||
|
auto & peekable_buf = assert_cast<PeekableReadBuffer &>(buf);
|
||||||
|
peekable_buf.setCheckpoint();
|
||||||
|
SCOPE_EXIT(peekable_buf.dropCheckpoint());
|
||||||
|
if (checkString(EMPTY_STRING, peekable_buf))
|
||||||
|
return true;
|
||||||
|
peekable_buf.rollbackToCheckpoint();
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
auto deserialize_nested_with_check = [&deserialize_nested](IColumn & column_, ReadBuffer & buf) -> ReturnType
|
||||||
|
{
|
||||||
|
auto & peekable_buf = assert_cast<PeekableReadBuffer &>(buf);
|
||||||
|
if constexpr (throw_exception)
|
||||||
|
deserialize_nested(column_, peekable_buf);
|
||||||
|
else if (!deserialize_nested(column_, peekable_buf))
|
||||||
|
return ReturnType(false);
|
||||||
|
|
||||||
|
if (unlikely(peekable_buf.hasUnreadData()))
|
||||||
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Incorrect state while parsing JSON: PeekableReadBuffer has unread data in own memory: {}", String(peekable_buf.position(), peekable_buf.available()));
|
||||||
|
|
||||||
|
return ReturnType(true);
|
||||||
|
};
|
||||||
|
|
||||||
|
PeekableReadBuffer peekable_buf(istr, true);
|
||||||
|
return do_deserialize(column, peekable_buf, check_for_empty_string, deserialize_nested_with_check);
|
||||||
|
}
|
||||||
|
|
||||||
|
template void deserializeEmpyStringAsDefaultOrNested<void, true>(IColumn & column, ReadBuffer & istr, const NestedDeserialize<void> & deserialize_nested);
|
||||||
|
template bool deserializeEmpyStringAsDefaultOrNested<bool, true>(IColumn & column, ReadBuffer & istr, const NestedDeserialize<bool> & deserialize_nested);
|
||||||
|
template bool deserializeEmpyStringAsDefaultOrNested<bool, false>(IColumn & column, ReadBuffer & istr, const NestedDeserialize<bool> & deserialize_nested);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -8,6 +8,7 @@
|
|||||||
#include <IO/Progress.h>
|
#include <IO/Progress.h>
|
||||||
#include <Core/NamesAndTypes.h>
|
#include <Core/NamesAndTypes.h>
|
||||||
#include <Common/Stopwatch.h>
|
#include <Common/Stopwatch.h>
|
||||||
|
#include <functional>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
@ -146,6 +147,16 @@ namespace JSONUtils
|
|||||||
|
|
||||||
bool skipUntilFieldInObject(ReadBuffer & in, const String & desired_field_name, const FormatSettings::JSON & settings);
|
bool skipUntilFieldInObject(ReadBuffer & in, const String & desired_field_name, const FormatSettings::JSON & settings);
|
||||||
void skipTheRestOfObject(ReadBuffer & in, const FormatSettings::JSON & settings);
|
void skipTheRestOfObject(ReadBuffer & in, const FormatSettings::JSON & settings);
|
||||||
|
|
||||||
|
template <typename ReturnType>
|
||||||
|
using NestedDeserialize = std::function<ReturnType(IColumn &, ReadBuffer &)>;
|
||||||
|
|
||||||
|
template <typename ReturnType, bool default_column_return_value = true>
|
||||||
|
ReturnType deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize<ReturnType> & deserialize_nested);
|
||||||
|
|
||||||
|
extern template void deserializeEmpyStringAsDefaultOrNested<void, true>(IColumn & column, ReadBuffer & istr, const NestedDeserialize<void> & deserialize_nested);
|
||||||
|
extern template bool deserializeEmpyStringAsDefaultOrNested<bool, true>(IColumn & column, ReadBuffer & istr, const NestedDeserialize<bool> & deserialize_nested);
|
||||||
|
extern template bool deserializeEmpyStringAsDefaultOrNested<bool, false>(IColumn & column, ReadBuffer & istr, const NestedDeserialize<bool> & deserialize_nested);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -10,6 +10,7 @@
|
|||||||
#include <Common/SensitiveDataMasker.h>
|
#include <Common/SensitiveDataMasker.h>
|
||||||
#include <Common/Macros.h>
|
#include <Common/Macros.h>
|
||||||
#include <Common/EventNotifier.h>
|
#include <Common/EventNotifier.h>
|
||||||
|
#include <Common/getNumberOfPhysicalCPUCores.h>
|
||||||
#include <Common/Stopwatch.h>
|
#include <Common/Stopwatch.h>
|
||||||
#include <Common/formatReadable.h>
|
#include <Common/formatReadable.h>
|
||||||
#include <Common/Throttler.h>
|
#include <Common/Throttler.h>
|
||||||
@ -121,7 +122,6 @@
|
|||||||
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
|
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
|
||||||
#include <base/defines.h>
|
#include <base/defines.h>
|
||||||
|
|
||||||
|
|
||||||
namespace fs = std::filesystem;
|
namespace fs = std::filesystem;
|
||||||
|
|
||||||
namespace ProfileEvents
|
namespace ProfileEvents
|
||||||
@ -164,6 +164,9 @@ namespace CurrentMetrics
|
|||||||
extern const Metric TablesLoaderForegroundThreadsActive;
|
extern const Metric TablesLoaderForegroundThreadsActive;
|
||||||
extern const Metric TablesLoaderForegroundThreadsScheduled;
|
extern const Metric TablesLoaderForegroundThreadsScheduled;
|
||||||
extern const Metric IOWriterThreadsScheduled;
|
extern const Metric IOWriterThreadsScheduled;
|
||||||
|
extern const Metric BuildVectorSimilarityIndexThreads;
|
||||||
|
extern const Metric BuildVectorSimilarityIndexThreadsActive;
|
||||||
|
extern const Metric BuildVectorSimilarityIndexThreadsScheduled;
|
||||||
extern const Metric AttachedTable;
|
extern const Metric AttachedTable;
|
||||||
extern const Metric AttachedView;
|
extern const Metric AttachedView;
|
||||||
extern const Metric AttachedDictionary;
|
extern const Metric AttachedDictionary;
|
||||||
@ -297,6 +300,8 @@ struct ContextSharedPart : boost::noncopyable
|
|||||||
mutable std::unique_ptr<ThreadPool> load_marks_threadpool; /// Threadpool for loading marks cache.
|
mutable std::unique_ptr<ThreadPool> load_marks_threadpool; /// Threadpool for loading marks cache.
|
||||||
mutable OnceFlag prefetch_threadpool_initialized;
|
mutable OnceFlag prefetch_threadpool_initialized;
|
||||||
mutable std::unique_ptr<ThreadPool> prefetch_threadpool; /// Threadpool for loading marks cache.
|
mutable std::unique_ptr<ThreadPool> prefetch_threadpool; /// Threadpool for loading marks cache.
|
||||||
|
mutable OnceFlag build_vector_similarity_index_threadpool_initialized;
|
||||||
|
mutable std::unique_ptr<ThreadPool> build_vector_similarity_index_threadpool; /// Threadpool for vector-similarity index creation.
|
||||||
mutable UncompressedCachePtr index_uncompressed_cache TSA_GUARDED_BY(mutex); /// The cache of decompressed blocks for MergeTree indices.
|
mutable UncompressedCachePtr index_uncompressed_cache TSA_GUARDED_BY(mutex); /// The cache of decompressed blocks for MergeTree indices.
|
||||||
mutable QueryCachePtr query_cache TSA_GUARDED_BY(mutex); /// Cache of query results.
|
mutable QueryCachePtr query_cache TSA_GUARDED_BY(mutex); /// Cache of query results.
|
||||||
mutable MarkCachePtr index_mark_cache TSA_GUARDED_BY(mutex); /// Cache of marks in compressed files of MergeTree indices.
|
mutable MarkCachePtr index_mark_cache TSA_GUARDED_BY(mutex); /// Cache of marks in compressed files of MergeTree indices.
|
||||||
@ -3297,6 +3302,21 @@ size_t Context::getPrefetchThreadpoolSize() const
|
|||||||
return config.getUInt(".prefetch_threadpool_pool_size", 100);
|
return config.getUInt(".prefetch_threadpool_pool_size", 100);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ThreadPool & Context::getBuildVectorSimilarityIndexThreadPool() const
|
||||||
|
{
|
||||||
|
callOnce(shared->build_vector_similarity_index_threadpool_initialized, [&] {
|
||||||
|
size_t pool_size = shared->server_settings.max_build_vector_similarity_index_thread_pool_size > 0
|
||||||
|
? shared->server_settings.max_build_vector_similarity_index_thread_pool_size
|
||||||
|
: getNumberOfPhysicalCPUCores();
|
||||||
|
shared->build_vector_similarity_index_threadpool = std::make_unique<ThreadPool>(
|
||||||
|
CurrentMetrics::BuildVectorSimilarityIndexThreads,
|
||||||
|
CurrentMetrics::BuildVectorSimilarityIndexThreadsActive,
|
||||||
|
CurrentMetrics::BuildVectorSimilarityIndexThreadsScheduled,
|
||||||
|
pool_size);
|
||||||
|
});
|
||||||
|
return *shared->build_vector_similarity_index_threadpool;
|
||||||
|
}
|
||||||
|
|
||||||
BackgroundSchedulePool & Context::getBufferFlushSchedulePool() const
|
BackgroundSchedulePool & Context::getBufferFlushSchedulePool() const
|
||||||
{
|
{
|
||||||
callOnce(shared->buffer_flush_schedule_pool_initialized, [&] {
|
callOnce(shared->buffer_flush_schedule_pool_initialized, [&] {
|
||||||
|
@ -1097,6 +1097,8 @@ public:
|
|||||||
/// and make a prefetch by putting a read task to threadpoolReader.
|
/// and make a prefetch by putting a read task to threadpoolReader.
|
||||||
size_t getPrefetchThreadpoolSize() const;
|
size_t getPrefetchThreadpoolSize() const;
|
||||||
|
|
||||||
|
ThreadPool & getBuildVectorSimilarityIndexThreadPool() const;
|
||||||
|
|
||||||
/// Settings for MergeTree background tasks stored in config.xml
|
/// Settings for MergeTree background tasks stored in config.xml
|
||||||
BackgroundTaskSchedulingSettings getBackgroundProcessingTaskSchedulingSettings() const;
|
BackgroundTaskSchedulingSettings getBackgroundProcessingTaskSchedulingSettings() const;
|
||||||
BackgroundTaskSchedulingSettings getBackgroundMoveTaskSchedulingSettings() const;
|
BackgroundTaskSchedulingSettings getBackgroundMoveTaskSchedulingSettings() const;
|
||||||
|
@ -1236,6 +1236,7 @@ IBlocksStreamPtr HashJoin::getNonJoinedBlocks(const Block & left_sample_block,
|
|||||||
|
|
||||||
void HashJoin::reuseJoinedData(const HashJoin & join)
|
void HashJoin::reuseJoinedData(const HashJoin & join)
|
||||||
{
|
{
|
||||||
|
have_compressed = join.have_compressed;
|
||||||
data = join.data;
|
data = join.data;
|
||||||
from_storage_join = true;
|
from_storage_join = true;
|
||||||
|
|
||||||
|
@ -99,6 +99,7 @@ namespace DB
|
|||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
{
|
{
|
||||||
extern const int QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS;
|
extern const int QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS;
|
||||||
|
extern const int QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE;
|
||||||
extern const int QUERY_CACHE_USED_WITH_SYSTEM_TABLE;
|
extern const int QUERY_CACHE_USED_WITH_SYSTEM_TABLE;
|
||||||
extern const int INTO_OUTFILE_NOT_ALLOWED;
|
extern const int INTO_OUTFILE_NOT_ALLOWED;
|
||||||
extern const int INVALID_TRANSACTION;
|
extern const int INVALID_TRANSACTION;
|
||||||
@ -1118,22 +1119,24 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
|
|||||||
&& settings.use_query_cache
|
&& settings.use_query_cache
|
||||||
&& !internal
|
&& !internal
|
||||||
&& client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY
|
&& client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY
|
||||||
/// Bug 67476: Avoid that the query cache stores truncated results if the query ran with a non-THROW overflow mode and hit a limit.
|
|
||||||
/// This is more workaround than a fix ... unfortunately it is hard to detect from the perspective of the query cache that the
|
|
||||||
/// query result is truncated.
|
|
||||||
&& (settings.read_overflow_mode == OverflowMode::THROW
|
|
||||||
&& settings.read_overflow_mode_leaf == OverflowMode::THROW
|
|
||||||
&& settings.group_by_overflow_mode == OverflowMode::THROW
|
|
||||||
&& settings.sort_overflow_mode == OverflowMode::THROW
|
|
||||||
&& settings.result_overflow_mode == OverflowMode::THROW
|
|
||||||
&& settings.timeout_overflow_mode == OverflowMode::THROW
|
|
||||||
&& settings.set_overflow_mode == OverflowMode::THROW
|
|
||||||
&& settings.join_overflow_mode == OverflowMode::THROW
|
|
||||||
&& settings.transfer_overflow_mode == OverflowMode::THROW
|
|
||||||
&& settings.distinct_overflow_mode == OverflowMode::THROW)
|
|
||||||
&& (ast->as<ASTSelectQuery>() || ast->as<ASTSelectWithUnionQuery>());
|
&& (ast->as<ASTSelectQuery>() || ast->as<ASTSelectWithUnionQuery>());
|
||||||
QueryCache::Usage query_cache_usage = QueryCache::Usage::None;
|
QueryCache::Usage query_cache_usage = QueryCache::Usage::None;
|
||||||
|
|
||||||
|
/// Bug 67476: If the query runs with a non-THROW overflow mode and hits a limit, the query cache will store a truncated result (if
|
||||||
|
/// enabled). This is incorrect. Unfortunately it is hard to detect from the perspective of the query cache that the query result
|
||||||
|
/// is truncated. Therefore throw an exception, to notify the user to disable either the query cache or use another overflow mode.
|
||||||
|
if (settings.use_query_cache && (settings.read_overflow_mode != OverflowMode::THROW
|
||||||
|
|| settings.read_overflow_mode_leaf != OverflowMode::THROW
|
||||||
|
|| settings.group_by_overflow_mode != OverflowMode::THROW
|
||||||
|
|| settings.sort_overflow_mode != OverflowMode::THROW
|
||||||
|
|| settings.result_overflow_mode != OverflowMode::THROW
|
||||||
|
|| settings.timeout_overflow_mode != OverflowMode::THROW
|
||||||
|
|| settings.set_overflow_mode != OverflowMode::THROW
|
||||||
|
|| settings.join_overflow_mode != OverflowMode::THROW
|
||||||
|
|| settings.transfer_overflow_mode != OverflowMode::THROW
|
||||||
|
|| settings.distinct_overflow_mode != OverflowMode::THROW))
|
||||||
|
throw Exception(ErrorCodes::QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE, "use_query_cache and overflow_mode != 'throw' cannot be used together");
|
||||||
|
|
||||||
/// If the query runs with "use_query_cache = 1", we first probe if the query cache already contains the query result (if yes:
|
/// If the query runs with "use_query_cache = 1", we first probe if the query cache already contains the query result (if yes:
|
||||||
/// return result from cache). If doesn't, we execute the query normally and write the result into the query cache. Both steps use a
|
/// return result from cache). If doesn't, we execute the query normally and write the result into the query cache. Both steps use a
|
||||||
/// hash of the AST, the current database and the settings as cache key. Unfortunately, the settings are in some places internally
|
/// hash of the AST, the current database and the settings as cache key. Unfortunately, the settings are in some places internally
|
||||||
|
@ -74,7 +74,8 @@ private:
|
|||||||
findMySQLFunctionSecretArguments();
|
findMySQLFunctionSecretArguments();
|
||||||
}
|
}
|
||||||
else if ((function.name == "s3") || (function.name == "cosn") || (function.name == "oss") ||
|
else if ((function.name == "s3") || (function.name == "cosn") || (function.name == "oss") ||
|
||||||
(function.name == "deltaLake") || (function.name == "hudi") || (function.name == "iceberg"))
|
(function.name == "deltaLake") || (function.name == "hudi") || (function.name == "iceberg") ||
|
||||||
|
(function.name == "gcs"))
|
||||||
{
|
{
|
||||||
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
|
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
|
||||||
findS3FunctionSecretArguments(/* is_cluster_function= */ false);
|
findS3FunctionSecretArguments(/* is_cluster_function= */ false);
|
||||||
|
@ -2009,33 +2009,6 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
|
|||||||
{
|
{
|
||||||
auto result = getAnalysisResult();
|
auto result = getAnalysisResult();
|
||||||
|
|
||||||
if (is_parallel_reading_from_replicas && context->canUseParallelReplicasOnInitiator()
|
|
||||||
&& context->getSettingsRef().parallel_replicas_local_plan)
|
|
||||||
{
|
|
||||||
CoordinationMode mode = CoordinationMode::Default;
|
|
||||||
switch (result.read_type)
|
|
||||||
{
|
|
||||||
case ReadFromMergeTree::ReadType::Default:
|
|
||||||
mode = CoordinationMode::Default;
|
|
||||||
break;
|
|
||||||
case ReadFromMergeTree::ReadType::InOrder:
|
|
||||||
mode = CoordinationMode::WithOrder;
|
|
||||||
break;
|
|
||||||
case ReadFromMergeTree::ReadType::InReverseOrder:
|
|
||||||
mode = CoordinationMode::ReverseOrder;
|
|
||||||
break;
|
|
||||||
case ReadFromMergeTree::ReadType::ParallelReplicas:
|
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Read type can't be ParallelReplicas on initiator");
|
|
||||||
}
|
|
||||||
|
|
||||||
chassert(number_of_current_replica.has_value());
|
|
||||||
chassert(all_ranges_callback.has_value());
|
|
||||||
|
|
||||||
/// initialize working set from local replica
|
|
||||||
all_ranges_callback.value()(
|
|
||||||
InitialAllRangesAnnouncement(mode, result.parts_with_ranges.getDescriptions(), number_of_current_replica.value()));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (enable_remove_parts_from_snapshot_optimization)
|
if (enable_remove_parts_from_snapshot_optimization)
|
||||||
{
|
{
|
||||||
/// Do not keep data parts in snapshot.
|
/// Do not keep data parts in snapshot.
|
||||||
|
@ -5,9 +5,11 @@
|
|||||||
#include <Columns/ColumnArray.h>
|
#include <Columns/ColumnArray.h>
|
||||||
#include <Common/BitHelpers.h>
|
#include <Common/BitHelpers.h>
|
||||||
#include <Common/formatReadable.h>
|
#include <Common/formatReadable.h>
|
||||||
|
#include <Common/getNumberOfPhysicalCPUCores.h>
|
||||||
#include <Common/logger_useful.h>
|
#include <Common/logger_useful.h>
|
||||||
#include <Common/typeid_cast.h>
|
#include <Common/typeid_cast.h>
|
||||||
#include <Core/Field.h>
|
#include <Core/Field.h>
|
||||||
|
#include <Core/ServerSettings.h>
|
||||||
#include <DataTypes/DataTypeArray.h>
|
#include <DataTypes/DataTypeArray.h>
|
||||||
#include <IO/ReadHelpers.h>
|
#include <IO/ReadHelpers.h>
|
||||||
#include <IO/WriteHelpers.h>
|
#include <IO/WriteHelpers.h>
|
||||||
@ -29,7 +31,6 @@ namespace DB
|
|||||||
|
|
||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
{
|
{
|
||||||
extern const int CANNOT_ALLOCATE_MEMORY;
|
|
||||||
extern const int FORMAT_VERSION_TOO_OLD;
|
extern const int FORMAT_VERSION_TOO_OLD;
|
||||||
extern const int ILLEGAL_COLUMN;
|
extern const int ILLEGAL_COLUMN;
|
||||||
extern const int INCORRECT_DATA;
|
extern const int INCORRECT_DATA;
|
||||||
@ -131,8 +132,7 @@ void USearchIndexWithSerialization::deserialize(ReadBuffer & istr)
|
|||||||
/// See the comment in MergeTreeIndexGranuleVectorSimilarity::deserializeBinary why we throw here
|
/// See the comment in MergeTreeIndexGranuleVectorSimilarity::deserializeBinary why we throw here
|
||||||
throw Exception(ErrorCodes::INCORRECT_DATA, "Could not load vector similarity index. Please drop the index and create it again. Error: {}", String(result.error.release()));
|
throw Exception(ErrorCodes::INCORRECT_DATA, "Could not load vector similarity index. Please drop the index and create it again. Error: {}", String(result.error.release()));
|
||||||
|
|
||||||
if (!try_reserve(limits()))
|
try_reserve(limits());
|
||||||
throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for usearch index");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
USearchIndexWithSerialization::Statistics USearchIndexWithSerialization::getStatistics() const
|
USearchIndexWithSerialization::Statistics USearchIndexWithSerialization::getStatistics() const
|
||||||
@ -270,20 +270,49 @@ void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & c
|
|||||||
throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column with vector similarity index must have equal length");
|
throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column with vector similarity index must have equal length");
|
||||||
|
|
||||||
/// Reserving space is mandatory
|
/// Reserving space is mandatory
|
||||||
if (!index->try_reserve(roundUpToPowerOfTwoOrZero(index->size() + rows)))
|
size_t max_thread_pool_size = Context::getGlobalContextInstance()->getServerSettings().max_build_vector_similarity_index_thread_pool_size;
|
||||||
throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for vector similarity index");
|
if (max_thread_pool_size == 0)
|
||||||
|
max_thread_pool_size = getNumberOfPhysicalCPUCores();
|
||||||
|
unum::usearch::index_limits_t limits(roundUpToPowerOfTwoOrZero(index->size() + rows), max_thread_pool_size);
|
||||||
|
index->reserve(limits);
|
||||||
|
|
||||||
for (size_t row = 0; row < rows; ++row)
|
/// Vector index creation is slooooow. Add the new rows in parallel. The threadpool is global to avoid oversubscription when multiple
|
||||||
|
/// indexes are build simultaneously (e.g. multiple merges run at the same time).
|
||||||
|
auto & thread_pool = Context::getGlobalContextInstance()->getBuildVectorSimilarityIndexThreadPool();
|
||||||
|
|
||||||
|
auto add_vector_to_index = [&](USearchIndex::vector_key_t key, size_t row, ThreadGroupPtr thread_group)
|
||||||
|
{
|
||||||
|
SCOPE_EXIT_SAFE(
|
||||||
|
if (thread_group)
|
||||||
|
CurrentThread::detachFromGroupIfNotDetached();
|
||||||
|
);
|
||||||
|
|
||||||
|
if (thread_group)
|
||||||
|
CurrentThread::attachToGroupIfDetached(thread_group);
|
||||||
|
|
||||||
|
/// add is thread-safe
|
||||||
|
if (auto result = index->add(key, &column_array_data_float_data[column_array_offsets[row - 1]]); !result)
|
||||||
{
|
{
|
||||||
if (auto result = index->add(static_cast<USearchIndex::vector_key_t>(index->size()), &column_array_data_float_data[column_array_offsets[row - 1]]); !result)
|
|
||||||
throw Exception(ErrorCodes::INCORRECT_DATA, "Could not add data to vector similarity index. Error: {}", String(result.error.release()));
|
throw Exception(ErrorCodes::INCORRECT_DATA, "Could not add data to vector similarity index. Error: {}", String(result.error.release()));
|
||||||
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
ProfileEvents::increment(ProfileEvents::USearchAddCount);
|
ProfileEvents::increment(ProfileEvents::USearchAddCount);
|
||||||
ProfileEvents::increment(ProfileEvents::USearchAddVisitedMembers, result.visited_members);
|
ProfileEvents::increment(ProfileEvents::USearchAddVisitedMembers, result.visited_members);
|
||||||
ProfileEvents::increment(ProfileEvents::USearchAddComputedDistances, result.computed_distances);
|
ProfileEvents::increment(ProfileEvents::USearchAddComputedDistances, result.computed_distances);
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
size_t index_size = index->size();
|
||||||
|
|
||||||
|
for (size_t row = 0; row < rows; ++row)
|
||||||
|
{
|
||||||
|
auto key = static_cast<USearchIndex::vector_key_t>(index_size + row);
|
||||||
|
auto task = [group = CurrentThread::getGroup(), &add_vector_to_index, key, row] { add_vector_to_index(key, row, group); };
|
||||||
|
thread_pool.scheduleOrThrowOnError(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
thread_pool.wait();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
#include <Storages/MergeTree/MergeTreeData.h>
|
#include <Storages/MergeTree/MergeTreeData.h>
|
||||||
#include <Storages/MergeTree/MergeTreePartsMover.h>
|
#include <Storages/MergeTree/MergeTreePartsMover.h>
|
||||||
#include <Storages/MergeTree/MergeTreeSettings.h>
|
#include <Storages/MergeTree/MergeTreeSettings.h>
|
||||||
|
#include <Common/FailPoint.h>
|
||||||
#include <Common/logger_useful.h>
|
#include <Common/logger_useful.h>
|
||||||
|
|
||||||
#include <set>
|
#include <set>
|
||||||
@ -15,6 +16,11 @@ namespace ErrorCodes
|
|||||||
extern const int DIRECTORY_ALREADY_EXISTS;
|
extern const int DIRECTORY_ALREADY_EXISTS;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
namespace FailPoints
|
||||||
|
{
|
||||||
|
extern const char stop_moving_part_before_swap_with_active[];
|
||||||
|
}
|
||||||
|
|
||||||
namespace
|
namespace
|
||||||
{
|
{
|
||||||
|
|
||||||
@ -226,6 +232,7 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me
|
|||||||
cloned_part.temporary_directory_lock = data->getTemporaryPartDirectoryHolder(part->name);
|
cloned_part.temporary_directory_lock = data->getTemporaryPartDirectoryHolder(part->name);
|
||||||
|
|
||||||
MutableDataPartStoragePtr cloned_part_storage;
|
MutableDataPartStoragePtr cloned_part_storage;
|
||||||
|
bool preserve_blobs = false;
|
||||||
if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication)
|
if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication)
|
||||||
{
|
{
|
||||||
/// Try zero-copy replication and fallback to default copy if it's not possible
|
/// Try zero-copy replication and fallback to default copy if it's not possible
|
||||||
@ -253,6 +260,7 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me
|
|||||||
if (zero_copy_part)
|
if (zero_copy_part)
|
||||||
{
|
{
|
||||||
/// FIXME for some reason we cannot just use this part, we have to re-create it through MergeTreeDataPartBuilder
|
/// FIXME for some reason we cannot just use this part, we have to re-create it through MergeTreeDataPartBuilder
|
||||||
|
preserve_blobs = true;
|
||||||
zero_copy_part->is_temp = false; /// Do not remove it in dtor
|
zero_copy_part->is_temp = false; /// Do not remove it in dtor
|
||||||
cloned_part_storage = zero_copy_part->getDataPartStoragePtr();
|
cloned_part_storage = zero_copy_part->getDataPartStoragePtr();
|
||||||
}
|
}
|
||||||
@ -272,7 +280,17 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me
|
|||||||
cloned_part.part = std::move(builder).withPartFormatFromDisk().build();
|
cloned_part.part = std::move(builder).withPartFormatFromDisk().build();
|
||||||
LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part.part->getDataPartStorage().getFullPath());
|
LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part.part->getDataPartStorage().getFullPath());
|
||||||
|
|
||||||
cloned_part.part->is_temp = data->allowRemoveStaleMovingParts();
|
cloned_part.part->is_temp = false;
|
||||||
|
if (data->allowRemoveStaleMovingParts())
|
||||||
|
{
|
||||||
|
cloned_part.part->is_temp = true;
|
||||||
|
/// Setting it in case connection to zookeeper is lost while moving
|
||||||
|
/// Otherwise part might be stuck in the moving directory due to the KEEPER_EXCEPTION in part's destructor
|
||||||
|
if (preserve_blobs)
|
||||||
|
cloned_part.part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::PRESERVE_BLOBS;
|
||||||
|
else
|
||||||
|
cloned_part.part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS;
|
||||||
|
}
|
||||||
cloned_part.part->loadColumnsChecksumsIndexes(true, true);
|
cloned_part.part->loadColumnsChecksumsIndexes(true, true);
|
||||||
cloned_part.part->loadVersionMetadata();
|
cloned_part.part->loadVersionMetadata();
|
||||||
cloned_part.part->modification_time = cloned_part.part->getDataPartStorage().getLastModified().epochTime();
|
cloned_part.part->modification_time = cloned_part.part->getDataPartStorage().getLastModified().epochTime();
|
||||||
@ -282,6 +300,8 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me
|
|||||||
|
|
||||||
void MergeTreePartsMover::swapClonedPart(TemporaryClonedPart & cloned_part) const
|
void MergeTreePartsMover::swapClonedPart(TemporaryClonedPart & cloned_part) const
|
||||||
{
|
{
|
||||||
|
/// Used to get some stuck parts in the moving directory by stopping moves while pause is active
|
||||||
|
FailPointInjection::pauseFailPoint(FailPoints::stop_moving_part_before_swap_with_active);
|
||||||
if (moves_blocker.isCancelled())
|
if (moves_blocker.isCancelled())
|
||||||
throw Exception(ErrorCodes::ABORTED, "Cancelled moving parts.");
|
throw Exception(ErrorCodes::ABORTED, "Cancelled moving parts.");
|
||||||
|
|
||||||
|
@ -15,6 +15,7 @@ namespace ErrorCodes
|
|||||||
{
|
{
|
||||||
extern const int SUPPORT_IS_DISABLED;
|
extern const int SUPPORT_IS_DISABLED;
|
||||||
extern const int REPLICA_STATUS_CHANGED;
|
extern const int REPLICA_STATUS_CHANGED;
|
||||||
|
extern const int LOGICAL_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
ReplicatedMergeTreeAttachThread::ReplicatedMergeTreeAttachThread(StorageReplicatedMergeTree & storage_)
|
ReplicatedMergeTreeAttachThread::ReplicatedMergeTreeAttachThread(StorageReplicatedMergeTree & storage_)
|
||||||
@ -117,6 +118,67 @@ void ReplicatedMergeTreeAttachThread::checkHasReplicaMetadataInZooKeeper(const z
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Int32 ReplicatedMergeTreeAttachThread::fixReplicaMetadataVersionIfNeeded(zkutil::ZooKeeperPtr zookeeper)
|
||||||
|
{
|
||||||
|
const String & zookeeper_path = storage.zookeeper_path;
|
||||||
|
const String & replica_path = storage.replica_path;
|
||||||
|
const bool replica_readonly = storage.is_readonly;
|
||||||
|
|
||||||
|
for (size_t i = 0; i != 2; ++i)
|
||||||
|
{
|
||||||
|
String replica_metadata_version_str;
|
||||||
|
const bool replica_metadata_version_exists = zookeeper->tryGet(replica_path + "/metadata_version", replica_metadata_version_str);
|
||||||
|
if (!replica_metadata_version_exists)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
const Int32 metadata_version = parse<Int32>(replica_metadata_version_str);
|
||||||
|
|
||||||
|
if (metadata_version != 0 || replica_readonly)
|
||||||
|
{
|
||||||
|
/// No need to fix anything
|
||||||
|
return metadata_version;
|
||||||
|
}
|
||||||
|
|
||||||
|
Coordination::Stat stat;
|
||||||
|
zookeeper->get(fs::path(zookeeper_path) / "metadata", &stat);
|
||||||
|
if (stat.version == 0)
|
||||||
|
{
|
||||||
|
/// No need to fix anything
|
||||||
|
return metadata_version;
|
||||||
|
}
|
||||||
|
|
||||||
|
ReplicatedMergeTreeQueue & queue = storage.queue;
|
||||||
|
queue.pullLogsToQueue(zookeeper);
|
||||||
|
if (queue.getStatus().metadata_alters_in_queue != 0)
|
||||||
|
{
|
||||||
|
LOG_DEBUG(log, "No need to update metadata_version as there are ALTER_METADATA entries in the queue");
|
||||||
|
return metadata_version;
|
||||||
|
}
|
||||||
|
|
||||||
|
const Coordination::Requests ops = {
|
||||||
|
zkutil::makeSetRequest(fs::path(replica_path) / "metadata_version", std::to_string(stat.version), 0),
|
||||||
|
zkutil::makeCheckRequest(fs::path(zookeeper_path) / "metadata", stat.version),
|
||||||
|
};
|
||||||
|
Coordination::Responses ops_responses;
|
||||||
|
const auto code = zookeeper->tryMulti(ops, ops_responses);
|
||||||
|
if (code == Coordination::Error::ZOK)
|
||||||
|
{
|
||||||
|
LOG_DEBUG(log, "Successfully set metadata_version to {}", stat.version);
|
||||||
|
return stat.version;
|
||||||
|
}
|
||||||
|
if (code != Coordination::Error::ZBADVERSION)
|
||||||
|
{
|
||||||
|
throw zkutil::KeeperException(code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Second attempt is only possible if metadata_version != 0 or metadata.version changed during the first attempt.
|
||||||
|
/// If metadata_version != 0, on second attempt we will return the new metadata_version.
|
||||||
|
/// If metadata.version changed, on second attempt we will either get metadata_version != 0 and return the new metadata_version or we will get metadata_alters_in_queue != 0 and return 0.
|
||||||
|
/// Either way, on second attempt this method should return.
|
||||||
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to fix replica metadata_version in ZooKeeper after two attempts");
|
||||||
|
}
|
||||||
|
|
||||||
void ReplicatedMergeTreeAttachThread::runImpl()
|
void ReplicatedMergeTreeAttachThread::runImpl()
|
||||||
{
|
{
|
||||||
storage.setZooKeeper();
|
storage.setZooKeeper();
|
||||||
@ -160,11 +222,11 @@ void ReplicatedMergeTreeAttachThread::runImpl()
|
|||||||
/// Just in case it was not removed earlier due to connection loss
|
/// Just in case it was not removed earlier due to connection loss
|
||||||
zookeeper->tryRemove(replica_path + "/flags/force_restore_data");
|
zookeeper->tryRemove(replica_path + "/flags/force_restore_data");
|
||||||
|
|
||||||
String replica_metadata_version;
|
const Int32 replica_metadata_version = fixReplicaMetadataVersionIfNeeded(zookeeper);
|
||||||
const bool replica_metadata_version_exists = zookeeper->tryGet(replica_path + "/metadata_version", replica_metadata_version);
|
const bool replica_metadata_version_exists = replica_metadata_version != -1;
|
||||||
if (replica_metadata_version_exists)
|
if (replica_metadata_version_exists)
|
||||||
{
|
{
|
||||||
storage.setInMemoryMetadata(metadata_snapshot->withMetadataVersion(parse<int>(replica_metadata_version)));
|
storage.setInMemoryMetadata(metadata_snapshot->withMetadataVersion(replica_metadata_version));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -48,6 +48,8 @@ private:
|
|||||||
void runImpl();
|
void runImpl();
|
||||||
|
|
||||||
void finalizeInitialization();
|
void finalizeInitialization();
|
||||||
|
|
||||||
|
Int32 fixReplicaMetadataVersionIfNeeded(zkutil::ZooKeeperPtr zookeeper);
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -2222,6 +2222,7 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const
|
|||||||
res.inserts_in_queue = 0;
|
res.inserts_in_queue = 0;
|
||||||
res.merges_in_queue = 0;
|
res.merges_in_queue = 0;
|
||||||
res.part_mutations_in_queue = 0;
|
res.part_mutations_in_queue = 0;
|
||||||
|
res.metadata_alters_in_queue = 0;
|
||||||
res.queue_oldest_time = 0;
|
res.queue_oldest_time = 0;
|
||||||
res.inserts_oldest_time = 0;
|
res.inserts_oldest_time = 0;
|
||||||
res.merges_oldest_time = 0;
|
res.merges_oldest_time = 0;
|
||||||
@ -2264,6 +2265,11 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const
|
|||||||
res.oldest_part_to_mutate_to = entry->new_part_name;
|
res.oldest_part_to_mutate_to = entry->new_part_name;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (entry->type == LogEntry::ALTER_METADATA)
|
||||||
|
{
|
||||||
|
++res.metadata_alters_in_queue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
|
@ -473,6 +473,7 @@ public:
|
|||||||
UInt32 inserts_in_queue;
|
UInt32 inserts_in_queue;
|
||||||
UInt32 merges_in_queue;
|
UInt32 merges_in_queue;
|
||||||
UInt32 part_mutations_in_queue;
|
UInt32 part_mutations_in_queue;
|
||||||
|
UInt32 metadata_alters_in_queue;
|
||||||
UInt32 queue_oldest_time;
|
UInt32 queue_oldest_time;
|
||||||
UInt32 inserts_oldest_time;
|
UInt32 inserts_oldest_time;
|
||||||
UInt32 merges_oldest_time;
|
UInt32 merges_oldest_time;
|
||||||
|
@ -277,7 +277,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs(
|
|||||||
|
|
||||||
if (has_valid_arguments)
|
if (has_valid_arguments)
|
||||||
{
|
{
|
||||||
if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 0)
|
if (!query.attach && is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 0)
|
||||||
{
|
{
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||||
"It's not allowed to specify explicit zookeeper_path and replica_name "
|
"It's not allowed to specify explicit zookeeper_path and replica_name "
|
||||||
@ -285,7 +285,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs(
|
|||||||
"specify them explicitly, enable setting "
|
"specify them explicitly, enable setting "
|
||||||
"database_replicated_allow_replicated_engine_arguments.");
|
"database_replicated_allow_replicated_engine_arguments.");
|
||||||
}
|
}
|
||||||
else if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 1)
|
else if (!query.attach && is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 1)
|
||||||
{
|
{
|
||||||
LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "It's not recommended to explicitly specify "
|
LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "It's not recommended to explicitly specify "
|
||||||
"zookeeper_path and replica_name in ReplicatedMergeTree arguments");
|
"zookeeper_path and replica_name in ReplicatedMergeTree arguments");
|
||||||
@ -305,7 +305,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs(
|
|||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Replica name must be a string literal{}", verbose_help_message);
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Replica name must be a string literal{}", verbose_help_message);
|
||||||
|
|
||||||
|
|
||||||
if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 2)
|
if (!query.attach && is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 2)
|
||||||
{
|
{
|
||||||
LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "Replacing user-provided ZooKeeper path and replica name ({}, {}) "
|
LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "Replacing user-provided ZooKeeper path and replica name ({}, {}) "
|
||||||
"with default arguments", zookeeper_path, replica_name);
|
"with default arguments", zookeeper_path, replica_name);
|
||||||
|
@ -42,6 +42,7 @@
|
|||||||
<multi_read>1</multi_read>
|
<multi_read>1</multi_read>
|
||||||
<check_not_exists>1</check_not_exists>
|
<check_not_exists>1</check_not_exists>
|
||||||
<create_if_not_exists>1</create_if_not_exists>
|
<create_if_not_exists>1</create_if_not_exists>
|
||||||
|
<remove_recursive>1</remove_recursive>
|
||||||
</feature_flags>
|
</feature_flags>
|
||||||
</keeper_server>
|
</keeper_server>
|
||||||
</clickhouse>
|
</clickhouse>
|
||||||
|
@ -105,7 +105,7 @@ setup_logs_replication
|
|||||||
|
|
||||||
clickhouse-client --query "SHOW DATABASES"
|
clickhouse-client --query "SHOW DATABASES"
|
||||||
clickhouse-client --query "CREATE DATABASE datasets"
|
clickhouse-client --query "CREATE DATABASE datasets"
|
||||||
clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql
|
clickhouse-client < /repo/tests/docker_scripts/create.sql
|
||||||
clickhouse-client --query "SHOW TABLES FROM datasets"
|
clickhouse-client --query "SHOW TABLES FROM datasets"
|
||||||
|
|
||||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||||
|
@ -62,7 +62,7 @@ start_server
|
|||||||
setup_logs_replication
|
setup_logs_replication
|
||||||
|
|
||||||
clickhouse-client --query "CREATE DATABASE datasets"
|
clickhouse-client --query "CREATE DATABASE datasets"
|
||||||
clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql
|
clickhouse-client < /repo/tests/docker_scripts/create.sql
|
||||||
clickhouse-client --query "SHOW TABLES FROM datasets"
|
clickhouse-client --query "SHOW TABLES FROM datasets"
|
||||||
|
|
||||||
clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test"
|
clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test"
|
||||||
|
@ -64,6 +64,7 @@ function configure()
|
|||||||
randomize_config_boolean_value multi_read keeper_port
|
randomize_config_boolean_value multi_read keeper_port
|
||||||
randomize_config_boolean_value check_not_exists keeper_port
|
randomize_config_boolean_value check_not_exists keeper_port
|
||||||
randomize_config_boolean_value create_if_not_exists keeper_port
|
randomize_config_boolean_value create_if_not_exists keeper_port
|
||||||
|
randomize_config_boolean_value remove_recursive keeper_port
|
||||||
fi
|
fi
|
||||||
|
|
||||||
sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
|
sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
|
||||||
|
@ -89,7 +89,6 @@ class Client:
|
|||||||
command = self.command[:]
|
command = self.command[:]
|
||||||
|
|
||||||
if stdin is None:
|
if stdin is None:
|
||||||
command += ["--multiquery"]
|
|
||||||
stdin = sql
|
stdin = sql
|
||||||
else:
|
else:
|
||||||
command += ["--query", sql]
|
command += ["--query", sql]
|
||||||
|
@ -393,6 +393,7 @@ def test_table_functions():
|
|||||||
f"azureBlobStorageCluster('test_shard_localhost', named_collection_2, connection_string = '{azure_conn_string}', container = 'cont', blob_path = 'test_simple_16.csv', format = 'CSV')",
|
f"azureBlobStorageCluster('test_shard_localhost', named_collection_2, connection_string = '{azure_conn_string}', container = 'cont', blob_path = 'test_simple_16.csv', format = 'CSV')",
|
||||||
f"azureBlobStorageCluster('test_shard_localhost', named_collection_2, storage_account_url = '{azure_storage_account_url}', container = 'cont', blob_path = 'test_simple_17.csv', account_name = '{azure_account_name}', account_key = '{azure_account_key}')",
|
f"azureBlobStorageCluster('test_shard_localhost', named_collection_2, storage_account_url = '{azure_storage_account_url}', container = 'cont', blob_path = 'test_simple_17.csv', account_name = '{azure_account_name}', account_key = '{azure_account_key}')",
|
||||||
f"iceberg('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')",
|
f"iceberg('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')",
|
||||||
|
f"gcs('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')",
|
||||||
]
|
]
|
||||||
|
|
||||||
def make_test_case(i):
|
def make_test_case(i):
|
||||||
|
@ -0,0 +1,33 @@
|
|||||||
|
<clickhouse>
|
||||||
|
<remote_servers>
|
||||||
|
<parallel_replicas>
|
||||||
|
<shard>
|
||||||
|
<internal_replication>false</internal_replication>
|
||||||
|
<replica>
|
||||||
|
<host>node0</host>
|
||||||
|
<port>9000</port>
|
||||||
|
</replica>
|
||||||
|
<replica>
|
||||||
|
<host>node1</host>
|
||||||
|
<port>9000</port>
|
||||||
|
</replica>
|
||||||
|
<replica>
|
||||||
|
<host>node2</host>
|
||||||
|
<port>9000</port>
|
||||||
|
</replica>
|
||||||
|
<replica>
|
||||||
|
<host>node3</host>
|
||||||
|
<port>9000</port>
|
||||||
|
</replica>
|
||||||
|
<replica>
|
||||||
|
<host>node4</host>
|
||||||
|
<port>9000</port>
|
||||||
|
</replica>
|
||||||
|
<replica>
|
||||||
|
<host>node5</host>
|
||||||
|
<port>9000</port>
|
||||||
|
</replica>
|
||||||
|
</shard>
|
||||||
|
</parallel_replicas>
|
||||||
|
</remote_servers>
|
||||||
|
</clickhouse>
|
@ -0,0 +1,73 @@
|
|||||||
|
import pytest
|
||||||
|
|
||||||
|
from helpers.cluster import ClickHouseCluster
|
||||||
|
|
||||||
|
cluster = ClickHouseCluster(__file__)
|
||||||
|
|
||||||
|
nodes = [
|
||||||
|
cluster.add_instance(
|
||||||
|
f"node{num}", main_configs=["configs/remote_servers.xml"], with_zookeeper=True
|
||||||
|
)
|
||||||
|
for num in range(6)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def start_cluster():
|
||||||
|
try:
|
||||||
|
cluster.start()
|
||||||
|
yield cluster
|
||||||
|
finally:
|
||||||
|
cluster.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def _create_tables(table_name):
|
||||||
|
for idx, node in enumerate(nodes):
|
||||||
|
node.query(
|
||||||
|
f"DROP TABLE IF EXISTS {table_name}",
|
||||||
|
settings={"database_atomic_wait_for_drop_and_detach_synchronously": True},
|
||||||
|
)
|
||||||
|
|
||||||
|
node.query(
|
||||||
|
f"""
|
||||||
|
CREATE TABLE {table_name} (value Int64)
|
||||||
|
Engine=ReplicatedMergeTree('/test_parallel_replicas/shard/{table_name}', '{idx}')
|
||||||
|
ORDER BY ()
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
nodes[0].query(
|
||||||
|
f"INSERT INTO {table_name} SELECT * FROM numbers(1000)",
|
||||||
|
settings={"insert_deduplicate": 0},
|
||||||
|
)
|
||||||
|
nodes[0].query(f"SYSTEM SYNC REPLICA ON CLUSTER 'parallel_replicas' {table_name}")
|
||||||
|
|
||||||
|
for idx, node in enumerate(nodes):
|
||||||
|
node.query("SYSTEM STOP REPLICATED SENDS")
|
||||||
|
# the same data on all nodes except for a single value
|
||||||
|
node.query(
|
||||||
|
f"INSERT INTO {table_name} VALUES ({idx})",
|
||||||
|
settings={"insert_deduplicate": 0},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# check that we use the state of data parts from the initiator node (for some sort of determinism of what is been read).
|
||||||
|
# currently it is implemented only when we build local plan for the initiator node (we aim to make this behavior default)
|
||||||
|
def test_initiator_snapshot_is_used_for_reading(start_cluster):
|
||||||
|
table_name = "t"
|
||||||
|
_create_tables(table_name)
|
||||||
|
|
||||||
|
for idx, node in enumerate(nodes):
|
||||||
|
expected = 499500 + idx # sum of all integers 0..999 + idx
|
||||||
|
assert (
|
||||||
|
node.query(
|
||||||
|
f"SELECT sum(value) FROM {table_name}",
|
||||||
|
settings={
|
||||||
|
"allow_experimental_parallel_reading_from_replicas": 2,
|
||||||
|
"max_parallel_replicas": 100,
|
||||||
|
"cluster_for_parallel_replicas": "parallel_replicas",
|
||||||
|
"parallel_replicas_local_plan": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
== f"{expected}\n"
|
||||||
|
)
|
46
tests/integration/test_remove_stale_moving_parts/config.xml
Normal file
46
tests/integration/test_remove_stale_moving_parts/config.xml
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
<clickhouse>
|
||||||
|
<remote_servers>
|
||||||
|
<cluster>
|
||||||
|
<shard>
|
||||||
|
<replica>
|
||||||
|
<host>ch1</host>
|
||||||
|
<port>9000</port>
|
||||||
|
</replica>
|
||||||
|
</shard>
|
||||||
|
</cluster>
|
||||||
|
</remote_servers>
|
||||||
|
<macros>
|
||||||
|
<shard>01</shard>
|
||||||
|
</macros>
|
||||||
|
<storage_configuration>
|
||||||
|
<disks>
|
||||||
|
<s3>
|
||||||
|
<type>s3</type>
|
||||||
|
<endpoint>http://minio1:9001/root/data/</endpoint>
|
||||||
|
<access_key_id>minio</access_key_id>
|
||||||
|
<secret_access_key>minio123</secret_access_key>
|
||||||
|
</s3>
|
||||||
|
</disks>
|
||||||
|
<policies>
|
||||||
|
<s3>
|
||||||
|
<volumes>
|
||||||
|
<default>
|
||||||
|
<disk>default</disk>
|
||||||
|
<perform_ttl_move_on_insert>False</perform_ttl_move_on_insert>
|
||||||
|
</default>
|
||||||
|
<s3>
|
||||||
|
<disk>s3</disk>
|
||||||
|
<perform_ttl_move_on_insert>False</perform_ttl_move_on_insert>
|
||||||
|
</s3>
|
||||||
|
</volumes>
|
||||||
|
<move_factor>0.0</move_factor>
|
||||||
|
</s3>
|
||||||
|
</policies>
|
||||||
|
</storage_configuration>
|
||||||
|
|
||||||
|
<merge_tree>
|
||||||
|
<allow_remote_fs_zero_copy_replication>true</allow_remote_fs_zero_copy_replication>
|
||||||
|
<storage_policy>s3</storage_policy>
|
||||||
|
</merge_tree>
|
||||||
|
<allow_remove_stale_moving_parts>true</allow_remove_stale_moving_parts>
|
||||||
|
</clickhouse>
|
117
tests/integration/test_remove_stale_moving_parts/test.py
Normal file
117
tests/integration/test_remove_stale_moving_parts/test.py
Normal file
@ -0,0 +1,117 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
import time
|
||||||
|
import pytest
|
||||||
|
from helpers.cluster import ClickHouseCluster
|
||||||
|
|
||||||
|
cluster = ClickHouseCluster(__file__)
|
||||||
|
ch1 = cluster.add_instance(
|
||||||
|
"ch1",
|
||||||
|
main_configs=[
|
||||||
|
"config.xml",
|
||||||
|
],
|
||||||
|
macros={"replica": "node1"},
|
||||||
|
with_zookeeper=True,
|
||||||
|
with_minio=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
DATABASE_NAME = "stale_moving_parts"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def started_cluster():
|
||||||
|
try:
|
||||||
|
cluster.start()
|
||||||
|
yield cluster
|
||||||
|
|
||||||
|
finally:
|
||||||
|
cluster.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def q(node, query):
|
||||||
|
return node.query(database=DATABASE_NAME, sql=query)
|
||||||
|
|
||||||
|
|
||||||
|
# .../disks/s3/store/
|
||||||
|
def get_table_path(node, table):
|
||||||
|
return (
|
||||||
|
node.query(
|
||||||
|
sql=f"SELECT data_paths FROM system.tables WHERE table = '{table}' and database = '{DATABASE_NAME}' LIMIT 1"
|
||||||
|
)
|
||||||
|
.strip('"\n[]')
|
||||||
|
.split(",")[1]
|
||||||
|
.strip("'")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def exec(node, cmd, path):
|
||||||
|
return node.exec_in_container(
|
||||||
|
[
|
||||||
|
"bash",
|
||||||
|
"-c",
|
||||||
|
f"{cmd} {path}",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_part_is_stuck(node, table_moving_path, moving_part):
|
||||||
|
num_tries = 5
|
||||||
|
while q(node, "SELECT part_name FROM system.moves").strip() != moving_part:
|
||||||
|
if num_tries == 0:
|
||||||
|
raise Exception("Part has not started to move")
|
||||||
|
num_tries -= 1
|
||||||
|
time.sleep(1)
|
||||||
|
num_tries = 5
|
||||||
|
while exec(node, "ls", table_moving_path).strip() != moving_part:
|
||||||
|
if num_tries == 0:
|
||||||
|
raise Exception("Part is not stuck in the moving directory")
|
||||||
|
num_tries -= 1
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_zookeeper_node_to_start(zk_nodes, timeout=60):
|
||||||
|
start = time.time()
|
||||||
|
while time.time() - start < timeout:
|
||||||
|
try:
|
||||||
|
for instance in zk_nodes:
|
||||||
|
conn = cluster.get_kazoo_client(instance)
|
||||||
|
conn.get_children("/")
|
||||||
|
print("All instances of ZooKeeper started")
|
||||||
|
return
|
||||||
|
except Exception as ex:
|
||||||
|
print(("Can't connect to ZooKeeper " + str(ex)))
|
||||||
|
time.sleep(0.5)
|
||||||
|
|
||||||
|
|
||||||
|
def test_remove_stale_moving_parts_without_zookeeper(started_cluster):
|
||||||
|
ch1.query(f"CREATE DATABASE IF NOT EXISTS {DATABASE_NAME}")
|
||||||
|
|
||||||
|
q(
|
||||||
|
ch1,
|
||||||
|
"CREATE TABLE test_remove ON CLUSTER cluster ( id UInt32 ) ENGINE ReplicatedMergeTree() ORDER BY id;",
|
||||||
|
)
|
||||||
|
|
||||||
|
table_moving_path = Path(get_table_path(ch1, "test_remove")) / "moving"
|
||||||
|
|
||||||
|
q(ch1, "SYSTEM ENABLE FAILPOINT stop_moving_part_before_swap_with_active")
|
||||||
|
q(ch1, "INSERT INTO test_remove SELECT number FROM numbers(100);")
|
||||||
|
moving_part = "all_0_0_0"
|
||||||
|
move_response = ch1.get_query_request(
|
||||||
|
sql=f"ALTER TABLE test_remove MOVE PART '{moving_part}' TO DISK 's3'",
|
||||||
|
database=DATABASE_NAME,
|
||||||
|
)
|
||||||
|
|
||||||
|
wait_part_is_stuck(ch1, table_moving_path, moving_part)
|
||||||
|
|
||||||
|
cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"])
|
||||||
|
# Stop moves in case table is not read-only yet
|
||||||
|
q(ch1, "SYSTEM STOP MOVES")
|
||||||
|
q(ch1, "SYSTEM DISABLE FAILPOINT stop_moving_part_before_swap_with_active")
|
||||||
|
|
||||||
|
assert "Cancelled moving parts" in move_response.get_error()
|
||||||
|
assert exec(ch1, "ls", table_moving_path).strip() == ""
|
||||||
|
|
||||||
|
cluster.start_zookeeper_nodes(["zoo1", "zoo2", "zoo3"])
|
||||||
|
wait_zookeeper_node_to_start(["zoo1", "zoo2", "zoo3"])
|
||||||
|
q(ch1, "SYSTEM START MOVES")
|
||||||
|
|
||||||
|
q(ch1, f"DROP TABLE test_remove")
|
@ -1549,3 +1549,19 @@ def test_all_groups_cluster(started_cluster):
|
|||||||
assert "bad_settings_node\ndummy_node\n" == bad_settings_node.query(
|
assert "bad_settings_node\ndummy_node\n" == bad_settings_node.query(
|
||||||
"select host_name from system.clusters where name='all_groups.db_cluster' order by host_name"
|
"select host_name from system.clusters where name='all_groups.db_cluster' order by host_name"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_detach_attach_table(started_cluster):
|
||||||
|
main_node.query("DROP DATABASE IF EXISTS detach_attach_db SYNC")
|
||||||
|
main_node.query(
|
||||||
|
"CREATE DATABASE detach_attach_db ENGINE = Replicated('/clickhouse/databases/detach_attach_db');"
|
||||||
|
)
|
||||||
|
main_node.query(
|
||||||
|
"CREATE TABLE detach_attach_db.detach_attach_table (k UInt64) ENGINE=ReplicatedMergeTree ORDER BY k;"
|
||||||
|
)
|
||||||
|
main_node.query("INSERT INTO detach_attach_db.detach_attach_table VALUES (1);")
|
||||||
|
main_node.query("DETACH TABLE detach_attach_db.detach_attach_table PERMANENTLY;")
|
||||||
|
main_node.query("ATTACH TABLE detach_attach_db.detach_attach_table;")
|
||||||
|
assert (
|
||||||
|
main_node.query("SELECT * FROM detach_attach_db.detach_attach_table;") == "1\n"
|
||||||
|
)
|
||||||
|
@ -427,7 +427,7 @@ do
|
|||||||
done
|
done
|
||||||
|
|
||||||
# for each query run, prepare array of metrics from query log
|
# for each query run, prepare array of metrics from query log
|
||||||
clickhouse-local --multiquery --query "
|
clickhouse-local --query "
|
||||||
create view query_runs as select * from file('analyze/query-runs.tsv', TSV,
|
create view query_runs as select * from file('analyze/query-runs.tsv', TSV,
|
||||||
'test text, query_index int, query_id text, version UInt8, time float');
|
'test text, query_index int, query_id text, version UInt8, time float');
|
||||||
|
|
||||||
@ -582,7 +582,7 @@ numactl --cpunodebind=all --membind=all numactl --show
|
|||||||
# If the available memory falls below 2 * size, GNU parallel will suspend some of the running jobs.
|
# If the available memory falls below 2 * size, GNU parallel will suspend some of the running jobs.
|
||||||
numactl --cpunodebind=all --membind=all parallel -v --joblog analyze/parallel-log.txt --memsuspend 15G --null < analyze/commands.txt 2>> analyze/errors.log
|
numactl --cpunodebind=all --membind=all parallel -v --joblog analyze/parallel-log.txt --memsuspend 15G --null < analyze/commands.txt 2>> analyze/errors.log
|
||||||
|
|
||||||
clickhouse-local --multiquery --query "
|
clickhouse-local --query "
|
||||||
-- Join the metric names back to the metric statistics we've calculated, and make
|
-- Join the metric names back to the metric statistics we've calculated, and make
|
||||||
-- a denormalized table of them -- statistics for all metrics for all queries.
|
-- a denormalized table of them -- statistics for all metrics for all queries.
|
||||||
-- The WITH, ARRAY JOIN and CROSS JOIN do not like each other:
|
-- The WITH, ARRAY JOIN and CROSS JOIN do not like each other:
|
||||||
@ -680,7 +680,7 @@ rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.ts
|
|||||||
cat analyze/errors.log >> report/errors.log ||:
|
cat analyze/errors.log >> report/errors.log ||:
|
||||||
cat profile-errors.log >> report/errors.log ||:
|
cat profile-errors.log >> report/errors.log ||:
|
||||||
|
|
||||||
clickhouse-local --multiquery --query "
|
clickhouse-local --query "
|
||||||
create view query_display_names as select * from
|
create view query_display_names as select * from
|
||||||
file('analyze/query-display-names.tsv', TSV,
|
file('analyze/query-display-names.tsv', TSV,
|
||||||
'test text, query_index int, query_display_name text')
|
'test text, query_index int, query_display_name text')
|
||||||
@ -981,7 +981,7 @@ create table all_query_metrics_tsv engine File(TSV, 'report/all-query-metrics.ts
|
|||||||
for version in {right,left}
|
for version in {right,left}
|
||||||
do
|
do
|
||||||
rm -rf data
|
rm -rf data
|
||||||
clickhouse-local --multiquery --query "
|
clickhouse-local --query "
|
||||||
create view query_profiles as
|
create view query_profiles as
|
||||||
with 0 as left, 1 as right
|
with 0 as left, 1 as right
|
||||||
select * from file('analyze/query-profiles.tsv', TSV,
|
select * from file('analyze/query-profiles.tsv', TSV,
|
||||||
@ -1151,7 +1151,7 @@ function report_metrics
|
|||||||
rm -rf metrics ||:
|
rm -rf metrics ||:
|
||||||
mkdir metrics
|
mkdir metrics
|
||||||
|
|
||||||
clickhouse-local --multiquery --query "
|
clickhouse-local --query "
|
||||||
create view right_async_metric_log as
|
create view right_async_metric_log as
|
||||||
select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes)
|
select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes)
|
||||||
;
|
;
|
||||||
@ -1211,7 +1211,7 @@ function upload_results
|
|||||||
# Prepare info for the CI checks table.
|
# Prepare info for the CI checks table.
|
||||||
rm -f ci-checks.tsv
|
rm -f ci-checks.tsv
|
||||||
|
|
||||||
clickhouse-local --multiquery --query "
|
clickhouse-local --query "
|
||||||
create view queries as select * from file('report/queries.tsv', TSVWithNamesAndTypes);
|
create view queries as select * from file('report/queries.tsv', TSVWithNamesAndTypes);
|
||||||
|
|
||||||
create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv')
|
create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv')
|
||||||
|
@ -5,4 +5,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CUR_DIR"/../shell_config.sh
|
. "$CUR_DIR"/../shell_config.sh
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*'
|
$CLICKHOUSE_CLIENT -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*'
|
||||||
|
@ -12,14 +12,14 @@ echo "
|
|||||||
DROP TABLE IF EXISTS rocksdb_race;
|
DROP TABLE IF EXISTS rocksdb_race;
|
||||||
CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key);
|
CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key);
|
||||||
INSERT INTO rocksdb_race SELECT '1_' || toString(number), number FROM numbers(100000);
|
INSERT INTO rocksdb_race SELECT '1_' || toString(number), number FROM numbers(100000);
|
||||||
" | $CLICKHOUSE_CLIENT -n
|
" | $CLICKHOUSE_CLIENT
|
||||||
|
|
||||||
function read_stat_thread()
|
function read_stat_thread()
|
||||||
{
|
{
|
||||||
while true; do
|
while true; do
|
||||||
echo "
|
echo "
|
||||||
SELECT * FROM system.rocksdb FORMAT Null;
|
SELECT * FROM system.rocksdb FORMAT Null;
|
||||||
" | $CLICKHOUSE_CLIENT -n
|
" | $CLICKHOUSE_CLIENT
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -29,7 +29,7 @@ function truncate_thread()
|
|||||||
sleep 3s;
|
sleep 3s;
|
||||||
echo "
|
echo "
|
||||||
TRUNCATE TABLE rocksdb_race;
|
TRUNCATE TABLE rocksdb_race;
|
||||||
" | $CLICKHOUSE_CLIENT -n
|
" | $CLICKHOUSE_CLIENT
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -12,7 +12,7 @@ opts=(
|
|||||||
--join_algorithm='parallel_hash'
|
--join_algorithm='parallel_hash'
|
||||||
)
|
)
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nq "
|
$CLICKHOUSE_CLIENT -q "
|
||||||
CREATE TABLE t1(a UInt32, b UInt32) ENGINE=MergeTree ORDER BY ();
|
CREATE TABLE t1(a UInt32, b UInt32) ENGINE=MergeTree ORDER BY ();
|
||||||
INSERT INTO t1 SELECT number, number FROM numbers_mt(1e6);
|
INSERT INTO t1 SELECT number, number FROM numbers_mt(1e6);
|
||||||
|
|
||||||
|
@ -5,12 +5,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CUR_DIR"/../shell_config.sh
|
. "$CUR_DIR"/../shell_config.sh
|
||||||
|
|
||||||
timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -nm -q "
|
timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -m -q "
|
||||||
SELECT sleep(1) FROM numbers(100) FORMAT Null;
|
SELECT sleep(1) FROM numbers(100) FORMAT Null;
|
||||||
SELECT 'FAIL';
|
SELECT 'FAIL';
|
||||||
"
|
"
|
||||||
|
|
||||||
timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -nm -q "
|
timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -m -q "
|
||||||
SELECT sleep(1) FROM numbers(100) FORMAT Null;
|
SELECT sleep(1) FROM numbers(100) FORMAT Null;
|
||||||
SELECT 'FAIL';
|
SELECT 'FAIL';
|
||||||
"
|
"
|
||||||
|
@ -16,7 +16,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
|||||||
|
|
||||||
$CLICKHOUSE_CLIENT --echo --query "SYSTEM DROP FILESYSTEM CACHE"
|
$CLICKHOUSE_CLIENT --echo --query "SYSTEM DROP FILESYSTEM CACHE"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state
|
$CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state
|
||||||
FROM
|
FROM
|
||||||
(
|
(
|
||||||
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
|
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
|
||||||
@ -37,7 +37,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
|||||||
|
|
||||||
$CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)"
|
$CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state
|
$CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state
|
||||||
FROM
|
FROM
|
||||||
(
|
(
|
||||||
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
|
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
|
||||||
@ -70,7 +70,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
|||||||
|
|
||||||
$CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100, 200)"
|
$CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100, 200)"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state
|
$CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state
|
||||||
FROM
|
FROM
|
||||||
(
|
(
|
||||||
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
|
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
|
||||||
@ -109,7 +109,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
|||||||
|
|
||||||
$CLICKHOUSE_CLIENT --echo --query "SYSTEM FLUSH LOGS"
|
$CLICKHOUSE_CLIENT --echo --query "SYSTEM FLUSH LOGS"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query "SELECT
|
$CLICKHOUSE_CLIENT --query "SELECT
|
||||||
query, ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read
|
query, ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read
|
||||||
FROM
|
FROM
|
||||||
system.query_log
|
system.query_log
|
||||||
|
@ -15,7 +15,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CURDIR"/../shell_config.sh
|
. "$CURDIR"/../shell_config.sh
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
DROP ROLE IF EXISTS test_role_02242;
|
DROP ROLE IF EXISTS test_role_02242;
|
||||||
CREATE ROLE test_role_02242;
|
CREATE ROLE test_role_02242;
|
||||||
"
|
"
|
||||||
|
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CURDIR"/../shell_config.sh
|
. "$CURDIR"/../shell_config.sh
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
DROP ROLE IF EXISTS test_role_02244;
|
DROP ROLE IF EXISTS test_role_02244;
|
||||||
CREATE ROLE test_role_02244;
|
CREATE ROLE test_role_02244;
|
||||||
DROP USER IF EXISTS kek_02243;
|
DROP USER IF EXISTS kek_02243;
|
||||||
|
@ -44,7 +44,7 @@ protobuf_info() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query "
|
$CLICKHOUSE_CLIENT --query "
|
||||||
DROP TABLE IF EXISTS $MAIN_TABLE;
|
DROP TABLE IF EXISTS $MAIN_TABLE;
|
||||||
DROP TABLE IF EXISTS $ROUNDTRIP_TABLE;
|
DROP TABLE IF EXISTS $ROUNDTRIP_TABLE;
|
||||||
DROP TABLE IF EXISTS $COMPATIBILITY_TABLE;
|
DROP TABLE IF EXISTS $COMPATIBILITY_TABLE;
|
||||||
@ -78,14 +78,14 @@ echo $SET_OUTPUT
|
|||||||
|
|
||||||
echo
|
echo
|
||||||
echo "Insert $INITIAL_INSERT_VALUES into table (Nullable(String), Int32):"
|
echo "Insert $INITIAL_INSERT_VALUES into table (Nullable(String), Int32):"
|
||||||
$CLICKHOUSE_CLIENT -n --query "
|
$CLICKHOUSE_CLIENT --query "
|
||||||
INSERT INTO $MAIN_TABLE VALUES $INITIAL_INSERT_VALUES;
|
INSERT INTO $MAIN_TABLE VALUES $INITIAL_INSERT_VALUES;
|
||||||
SELECT * FROM $MAIN_TABLE;
|
SELECT * FROM $MAIN_TABLE;
|
||||||
"
|
"
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo "Protobuf representation of the second row:"
|
echo "Protobuf representation of the second row:"
|
||||||
$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH"
|
$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH"
|
||||||
hexdump -C $BINARY_FILE_PATH
|
hexdump -C $BINARY_FILE_PATH
|
||||||
|
|
||||||
echo
|
echo
|
||||||
@ -101,12 +101,12 @@ hexdump -C $MESSAGE_FILE_PATH
|
|||||||
|
|
||||||
echo
|
echo
|
||||||
echo "Insert proto message into table (Nullable(String), Int32):"
|
echo "Insert proto message into table (Nullable(String), Int32):"
|
||||||
$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH"
|
$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH"
|
||||||
$CLICKHOUSE_CLIENT --query "SELECT * FROM $ROUNDTRIP_TABLE"
|
$CLICKHOUSE_CLIENT --query "SELECT * FROM $ROUNDTRIP_TABLE"
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo "Proto output of the table using Google wrapper:"
|
echo "Proto output of the table using Google wrapper:"
|
||||||
$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH"
|
$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH"
|
||||||
hexdump -C $BINARY_FILE_PATH
|
hexdump -C $BINARY_FILE_PATH
|
||||||
|
|
||||||
echo
|
echo
|
||||||
@ -124,14 +124,14 @@ echo
|
|||||||
echo "Insert $MULTI_WRAPPER_VALUES and reinsert using Google wrappers into:"
|
echo "Insert $MULTI_WRAPPER_VALUES and reinsert using Google wrappers into:"
|
||||||
echo "Table (Nullable(Int32), Nullable(Int32), Int32):"
|
echo "Table (Nullable(Int32), Nullable(Int32), Int32):"
|
||||||
$CLICKHOUSE_CLIENT --query "INSERT INTO $MULTI_TABLE VALUES $MULTI_WRAPPER_VALUES"
|
$CLICKHOUSE_CLIENT --query "INSERT INTO $MULTI_TABLE VALUES $MULTI_WRAPPER_VALUES"
|
||||||
$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH"
|
$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH"
|
||||||
$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH"
|
$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH"
|
||||||
$CLICKHOUSE_CLIENT --query "SELECT * FROM $MULTI_TABLE"
|
$CLICKHOUSE_CLIENT --query "SELECT * FROM $MULTI_TABLE"
|
||||||
|
|
||||||
rm "$BINARY_FILE_PATH"
|
rm "$BINARY_FILE_PATH"
|
||||||
rm "$MESSAGE_FILE_PATH"
|
rm "$MESSAGE_FILE_PATH"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query "
|
$CLICKHOUSE_CLIENT --query "
|
||||||
DROP TABLE $MAIN_TABLE;
|
DROP TABLE $MAIN_TABLE;
|
||||||
DROP TABLE $ROUNDTRIP_TABLE;
|
DROP TABLE $ROUNDTRIP_TABLE;
|
||||||
DROP TABLE $COMPATIBILITY_TABLE;
|
DROP TABLE $COMPATIBILITY_TABLE;
|
||||||
|
@ -11,7 +11,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
|||||||
echo "Using storage policy: $STORAGE_POLICY"
|
echo "Using storage policy: $STORAGE_POLICY"
|
||||||
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286"
|
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_02286 (key UInt32, value String)
|
$CLICKHOUSE_CLIENT --query "CREATE TABLE test_02286 (key UInt32, value String)
|
||||||
Engine=MergeTree()
|
Engine=MergeTree()
|
||||||
ORDER BY key
|
ORDER BY key
|
||||||
SETTINGS storage_policy='$STORAGE_POLICY', min_bytes_for_wide_part = 10485760"
|
SETTINGS storage_policy='$STORAGE_POLICY', min_bytes_for_wide_part = 10485760"
|
||||||
@ -38,7 +38,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
|||||||
$CLICKHOUSE_CLIENT --query "SELECT * FROM test_02286 FORMAT Null"
|
$CLICKHOUSE_CLIENT --query "SELECT * FROM test_02286 FORMAT Null"
|
||||||
$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
|
$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query "SELECT count()
|
$CLICKHOUSE_CLIENT --query "SELECT count()
|
||||||
FROM (
|
FROM (
|
||||||
SELECT
|
SELECT
|
||||||
arrayJoin(cache_paths) AS cache_path,
|
arrayJoin(cache_paths) AS cache_path,
|
||||||
@ -54,7 +54,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
|||||||
$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
|
$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT --query "SELECT cache_path FROM system.filesystem_cache"
|
$CLICKHOUSE_CLIENT --query "SELECT cache_path FROM system.filesystem_cache"
|
||||||
$CLICKHOUSE_CLIENT -n --query "SELECT cache_path, local_path
|
$CLICKHOUSE_CLIENT --query "SELECT cache_path, local_path
|
||||||
FROM (
|
FROM (
|
||||||
SELECT
|
SELECT
|
||||||
arrayJoin(cache_paths) AS cache_path,
|
arrayJoin(cache_paths) AS cache_path,
|
||||||
|
@ -23,7 +23,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; d
|
|||||||
ORDER BY tuple()
|
ORDER BY tuple()
|
||||||
SETTINGS storage_policy = '$STORAGE_POLICY'" > /dev/null
|
SETTINGS storage_policy = '$STORAGE_POLICY'" > /dev/null
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 -n --query "INSERT INTO test_02313
|
$CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 --query "INSERT INTO test_02313
|
||||||
SELECT * FROM
|
SELECT * FROM
|
||||||
generateRandom('id Int32, val String')
|
generateRandom('id Int32, val String')
|
||||||
LIMIT 100000"
|
LIMIT 100000"
|
||||||
|
@ -9,7 +9,7 @@ function check_refcnt_for_table()
|
|||||||
{
|
{
|
||||||
local table=$1 && shift
|
local table=$1 && shift
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
system stop merges $table;
|
system stop merges $table;
|
||||||
-- cleanup thread may hold the parts lock
|
-- cleanup thread may hold the parts lock
|
||||||
system stop cleanup $table;
|
system stop cleanup $table;
|
||||||
@ -66,14 +66,14 @@ function check_refcnt_for_table()
|
|||||||
|
|
||||||
# NOTE: index_granularity=1 to cancel ASAP
|
# NOTE: index_granularity=1 to cancel ASAP
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
drop table if exists data_02340;
|
drop table if exists data_02340;
|
||||||
create table data_02340 (key Int, part Int) engine=MergeTree() partition by part order by key settings index_granularity=1;
|
create table data_02340 (key Int, part Int) engine=MergeTree() partition by part order by key settings index_granularity=1;
|
||||||
" || exit 1
|
" || exit 1
|
||||||
check_refcnt_for_table data_02340
|
check_refcnt_for_table data_02340
|
||||||
$CLICKHOUSE_CLIENT -q "drop table data_02340 sync"
|
$CLICKHOUSE_CLIENT -q "drop table data_02340 sync"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
drop table if exists data_02340_rep sync;
|
drop table if exists data_02340_rep sync;
|
||||||
create table data_02340_rep (key Int, part Int) engine=ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') partition by part order by key settings index_granularity=1;
|
create table data_02340_rep (key Int, part Int) engine=ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') partition by part order by key settings index_granularity=1;
|
||||||
" || exit 1
|
" || exit 1
|
||||||
|
@ -7,14 +7,14 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
|
|
||||||
disk_name="02344_describe_cache_test"
|
disk_name="02344_describe_cache_test"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm --query """
|
$CLICKHOUSE_CLIENT -m --query """
|
||||||
DROP TABLE IF EXISTS test;
|
DROP TABLE IF EXISTS test;
|
||||||
CREATE TABLE test (a Int32, b String)
|
CREATE TABLE test (a Int32, b String)
|
||||||
ENGINE = MergeTree() ORDER BY tuple()
|
ENGINE = MergeTree() ORDER BY tuple()
|
||||||
SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = '$disk_name', disk = 's3_disk', load_metadata_asynchronously = 0);
|
SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = '$disk_name', disk = 's3_disk', load_metadata_asynchronously = 0);
|
||||||
"""
|
"""
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm --query """
|
$CLICKHOUSE_CLIENT -m --query """
|
||||||
SELECT count() FROM system.disks WHERE name = '$disk_name'
|
SELECT count() FROM system.disks WHERE name = '$disk_name'
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@ -24,7 +24,7 @@ function wait_query_by_id_started()
|
|||||||
# wait for query to be started
|
# wait for query to be started
|
||||||
while [ "$($CLICKHOUSE_CLIENT "$@" -q "select count() from system.processes where query_id = '$query_id'")" -ne 1 ]; do
|
while [ "$($CLICKHOUSE_CLIENT "$@" -q "select count() from system.processes where query_id = '$query_id'")" -ne 1 ]; do
|
||||||
if [ "$(
|
if [ "$(
|
||||||
$CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -nm -q "
|
$CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -m -q "
|
||||||
system flush logs;
|
system flush logs;
|
||||||
|
|
||||||
select count() from system.query_log
|
select count() from system.query_log
|
||||||
@ -52,7 +52,7 @@ $CLICKHOUSE_CLIENT -q "CREATE DATABASE ${CLICKHOUSE_DATABASE}_ordinary Engine=Or
|
|||||||
# debug build on CI, so if this will happen, then DROP query will be
|
# debug build on CI, so if this will happen, then DROP query will be
|
||||||
# finished instantly, and to avoid flakiness we will retry in this case
|
# finished instantly, and to avoid flakiness we will retry in this case
|
||||||
while :; do
|
while :; do
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}_ordinary.data_02352;
|
DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}_ordinary.data_02352;
|
||||||
CREATE TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 (key Int) Engine=Null();
|
CREATE TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 (key Int) Engine=Null();
|
||||||
"
|
"
|
||||||
|
@ -9,13 +9,13 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CUR_DIR"/../shell_config.sh
|
. "$CUR_DIR"/../shell_config.sh
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "ATTACH TABLE mv" |& {
|
$CLICKHOUSE_CLIENT -m -q "ATTACH TABLE mv" |& {
|
||||||
# CANNOT_GET_CREATE_TABLE_QUERY -- ATTACH TABLE IF EXISTS
|
# CANNOT_GET_CREATE_TABLE_QUERY -- ATTACH TABLE IF EXISTS
|
||||||
# TABLE_ALREADY_EXISTS -- ATTACH TABLE IF NOT EXISTS
|
# TABLE_ALREADY_EXISTS -- ATTACH TABLE IF NOT EXISTS
|
||||||
grep -F -m1 Exception | grep -v -e CANNOT_GET_CREATE_TABLE_QUERY -e TABLE_ALREADY_EXISTS
|
grep -F -m1 Exception | grep -v -e CANNOT_GET_CREATE_TABLE_QUERY -e TABLE_ALREADY_EXISTS
|
||||||
}
|
}
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
DROP TABLE IF EXISTS null;
|
DROP TABLE IF EXISTS null;
|
||||||
CREATE TABLE null (key Int) ENGINE = Null;
|
CREATE TABLE null (key Int) ENGINE = Null;
|
||||||
DROP TABLE IF EXISTS mv;
|
DROP TABLE IF EXISTS mv;
|
||||||
|
@ -27,7 +27,7 @@ function insert()
|
|||||||
|
|
||||||
function check_span()
|
function check_span()
|
||||||
{
|
{
|
||||||
${CLICKHOUSE_CLIENT} -nq "
|
${CLICKHOUSE_CLIENT} -q "
|
||||||
SYSTEM FLUSH LOGS;
|
SYSTEM FLUSH LOGS;
|
||||||
|
|
||||||
SELECT operation_name,
|
SELECT operation_name,
|
||||||
@ -50,7 +50,7 @@ ${CLICKHOUSE_CLIENT} -nq "
|
|||||||
# $2 - value of distributed_foreground_insert
|
# $2 - value of distributed_foreground_insert
|
||||||
function check_span_kind()
|
function check_span_kind()
|
||||||
{
|
{
|
||||||
${CLICKHOUSE_CLIENT} -nq "
|
${CLICKHOUSE_CLIENT} -q "
|
||||||
SYSTEM FLUSH LOGS;
|
SYSTEM FLUSH LOGS;
|
||||||
|
|
||||||
SELECT count()
|
SELECT count()
|
||||||
@ -65,7 +65,7 @@ ${CLICKHOUSE_CLIENT} -nq "
|
|||||||
#
|
#
|
||||||
# Prepare tables for tests
|
# Prepare tables for tests
|
||||||
#
|
#
|
||||||
${CLICKHOUSE_CLIENT} -nq "
|
${CLICKHOUSE_CLIENT} -q "
|
||||||
DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.dist_opentelemetry;
|
DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.dist_opentelemetry;
|
||||||
DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.local_opentelemetry;
|
DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.local_opentelemetry;
|
||||||
|
|
||||||
@ -122,7 +122,7 @@ check_span_kind $trace_id 'CLIENT'
|
|||||||
#
|
#
|
||||||
# Cleanup
|
# Cleanup
|
||||||
#
|
#
|
||||||
${CLICKHOUSE_CLIENT} -nq "
|
${CLICKHOUSE_CLIENT} -q "
|
||||||
DROP TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry;
|
DROP TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry;
|
||||||
DROP TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry;
|
DROP TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry;
|
||||||
"
|
"
|
||||||
|
@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02419_test SYNC;"
|
|||||||
|
|
||||||
test_primary_key()
|
test_primary_key()
|
||||||
{
|
{
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
CREATE TABLE 02419_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY($1);
|
CREATE TABLE 02419_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY($1);
|
||||||
INSERT INTO 02419_test VALUES (1, 1.1), (2, 2.2);
|
INSERT INTO 02419_test VALUES (1, 1.1), (2, 2.2);
|
||||||
SELECT value FROM 02419_test WHERE key = 1;
|
SELECT value FROM 02419_test WHERE key = 1;
|
||||||
|
@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
. "$CURDIR"/replication.lib
|
. "$CURDIR"/replication.lib
|
||||||
|
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n -q "
|
$CLICKHOUSE_CLIENT -q "
|
||||||
DROP TABLE IF EXISTS alter_table0;
|
DROP TABLE IF EXISTS alter_table0;
|
||||||
DROP TABLE IF EXISTS alter_table1;
|
DROP TABLE IF EXISTS alter_table1;
|
||||||
|
|
||||||
|
@ -21,7 +21,7 @@ wait_for_number_of_parts() {
|
|||||||
echo "$res"
|
echo "$res"
|
||||||
}
|
}
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
DROP TABLE IF EXISTS test_without_merge;
|
DROP TABLE IF EXISTS test_without_merge;
|
||||||
DROP TABLE IF EXISTS test_with_merge;
|
DROP TABLE IF EXISTS test_with_merge;
|
||||||
|
|
||||||
@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;"
|
|||||||
|
|
||||||
wait_for_number_of_parts 'test_without_merge' 1 10
|
wait_for_number_of_parts 'test_without_merge' 1 10
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
DROP TABLE test_without_merge;
|
DROP TABLE test_without_merge;
|
||||||
|
|
||||||
SELECT 'With merge any part range';
|
SELECT 'With merge any part range';
|
||||||
@ -47,7 +47,7 @@ INSERT INTO test_with_merge SELECT 3;"
|
|||||||
|
|
||||||
wait_for_number_of_parts 'test_with_merge' 1 100
|
wait_for_number_of_parts 'test_with_merge' 1 100
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
DROP TABLE test_with_merge;
|
DROP TABLE test_with_merge;
|
||||||
|
|
||||||
SELECT 'With merge partition only';
|
SELECT 'With merge partition only';
|
||||||
@ -60,7 +60,7 @@ INSERT INTO test_with_merge SELECT 3;"
|
|||||||
|
|
||||||
wait_for_number_of_parts 'test_with_merge' 1 100
|
wait_for_number_of_parts 'test_with_merge' 1 100
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one
|
SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one
|
||||||
SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_with_merge' AND active;
|
SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_with_merge' AND active;
|
||||||
|
|
||||||
|
@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CURDIR"/../shell_config.sh
|
. "$CURDIR"/../shell_config.sh
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nq "
|
$CLICKHOUSE_CLIENT -q "
|
||||||
-- Limit S3 PUT request per second rate
|
-- Limit S3 PUT request per second rate
|
||||||
SET s3_max_put_rps = 2;
|
SET s3_max_put_rps = 2;
|
||||||
SET s3_max_put_burst = 1;
|
SET s3_max_put_burst = 1;
|
||||||
|
@ -23,23 +23,3 @@ Row 1:
|
|||||||
x: 1
|
x: 1
|
||||||
2
|
2
|
||||||
-- Bug 67476: Queries with overflow mode != throw must not be cached by the query cache
|
-- Bug 67476: Queries with overflow mode != throw must not be cached by the query cache
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
0
|
|
||||||
|
@ -43,25 +43,15 @@ DROP TABLE IF EXISTS tab;
|
|||||||
CREATE TABLE tab(c UInt64) ENGINE = Memory;
|
CREATE TABLE tab(c UInt64) ENGINE = Memory;
|
||||||
|
|
||||||
SYSTEM DROP QUERY CACHE;
|
SYSTEM DROP QUERY CACHE;
|
||||||
SELECT sum(c) FROM tab SETTINGS read_overflow_mode = 'break', use_query_cache = 1;
|
SELECT sum(c) FROM tab SETTINGS read_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT count(*) from system.query_cache;
|
SELECT sum(c) FROM tab SETTINGS read_overflow_mode_leaf = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT sum(c) FROM tab SETTINGS read_overflow_mode_leaf = 'break', use_query_cache = 1;
|
SELECT sum(c) FROM tab SETTINGS group_by_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT count(*) from system.query_cache;
|
SELECT sum(c) FROM tab SETTINGS sort_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT sum(c) FROM tab SETTINGS group_by_overflow_mode = 'break', use_query_cache = 1;
|
SELECT sum(c) FROM tab SETTINGS result_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT count(*) from system.query_cache;
|
SELECT sum(c) FROM tab SETTINGS timeout_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT sum(c) FROM tab SETTINGS sort_overflow_mode = 'break', use_query_cache = 1;
|
SELECT sum(c) FROM tab SETTINGS set_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT count(*) from system.query_cache;
|
SELECT sum(c) FROM tab SETTINGS join_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT sum(c) FROM tab SETTINGS result_overflow_mode = 'break', use_query_cache = 1;
|
SELECT sum(c) FROM tab SETTINGS transfer_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT count(*) from system.query_cache;
|
SELECT sum(c) FROM tab SETTINGS distinct_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE }
|
||||||
SELECT sum(c) FROM tab SETTINGS timeout_overflow_mode = 'break', use_query_cache = 1;
|
|
||||||
SELECT count(*) from system.query_cache;
|
|
||||||
SELECT sum(c) FROM tab SETTINGS set_overflow_mode = 'break', use_query_cache = 1;
|
|
||||||
SELECT count(*) from system.query_cache;
|
|
||||||
SELECT sum(c) FROM tab SETTINGS join_overflow_mode = 'break', use_query_cache = 1;
|
|
||||||
SELECT count(*) from system.query_cache;
|
|
||||||
SELECT sum(c) FROM tab SETTINGS transfer_overflow_mode = 'break', use_query_cache = 1;
|
|
||||||
SELECT count(*) from system.query_cache;
|
|
||||||
SELECT sum(c) FROM tab SETTINGS distinct_overflow_mode = 'break', use_query_cache = 1;
|
|
||||||
SELECT count(*) from system.query_cache;
|
|
||||||
|
|
||||||
SYSTEM DROP QUERY CACHE;
|
SYSTEM DROP QUERY CACHE;
|
||||||
|
@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CUR_DIR"/../shell_config.sh
|
. "$CUR_DIR"/../shell_config.sh
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query "
|
$CLICKHOUSE_CLIENT --query "
|
||||||
DROP TABLE IF EXISTS wikistat1 SYNC;
|
DROP TABLE IF EXISTS wikistat1 SYNC;
|
||||||
DROP TABLE IF EXISTS wikistat2 SYNC;
|
DROP TABLE IF EXISTS wikistat2 SYNC;
|
||||||
"
|
"
|
||||||
@ -60,7 +60,7 @@ wait
|
|||||||
$CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat1 WHERE NOT ignore(*)"
|
$CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat1 WHERE NOT ignore(*)"
|
||||||
$CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat2 WHERE NOT ignore(*)"
|
$CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat2 WHERE NOT ignore(*)"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query "
|
$CLICKHOUSE_CLIENT --query "
|
||||||
DROP TABLE IF EXISTS wikistat1 SYNC;
|
DROP TABLE IF EXISTS wikistat1 SYNC;
|
||||||
DROP TABLE IF EXISTS wikistat2 SYNC;
|
DROP TABLE IF EXISTS wikistat2 SYNC;
|
||||||
"
|
"
|
||||||
|
@ -11,7 +11,7 @@ cp $CURDIR/data_ua_parser/os.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/
|
|||||||
cp $CURDIR/data_ua_parser/browser.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/
|
cp $CURDIR/data_ua_parser/browser.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/
|
||||||
cp $CURDIR/data_ua_parser/device.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/
|
cp $CURDIR/data_ua_parser/device.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
drop dictionary if exists regexp_os;
|
drop dictionary if exists regexp_os;
|
||||||
drop dictionary if exists regexp_browser;
|
drop dictionary if exists regexp_browser;
|
||||||
drop dictionary if exists regexp_device;
|
drop dictionary if exists regexp_device;
|
||||||
@ -61,10 +61,10 @@ create table user_agents
|
|||||||
Engine = Log();
|
Engine = Log();
|
||||||
"
|
"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
insert into user_agents select ua from input('ua String') FORMAT LineAsString" < $CURDIR/data_ua_parser/useragents.txt
|
insert into user_agents select ua from input('ua String') FORMAT LineAsString" < $CURDIR/data_ua_parser/useragents.txt
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
select ua, device,
|
select ua, device,
|
||||||
concat(tupleElement(browser, 1), ' ', tupleElement(browser, 2), '.', tupleElement(browser, 3)) as browser ,
|
concat(tupleElement(browser, 1), ' ', tupleElement(browser, 2), '.', tupleElement(browser, 3)) as browser ,
|
||||||
concat(tupleElement(os, 1), ' ', tupleElement(os, 2), '.', tupleElement(os, 3), '.', tupleElement(os, 4)) as os
|
concat(tupleElement(os, 1), ' ', tupleElement(os, 2), '.', tupleElement(os, 3), '.', tupleElement(os, 4)) as os
|
||||||
@ -74,7 +74,7 @@ from (
|
|||||||
dictGet('regexp_device', 'device_replacement', ua) device from user_agents) order by ua;
|
dictGet('regexp_device', 'device_replacement', ua) device from user_agents) order by ua;
|
||||||
"
|
"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
drop dictionary if exists regexp_os;
|
drop dictionary if exists regexp_os;
|
||||||
drop dictionary if exists regexp_browser;
|
drop dictionary if exists regexp_browser;
|
||||||
drop dictionary if exists regexp_device;
|
drop dictionary if exists regexp_device;
|
||||||
|
@ -27,7 +27,7 @@ cat > "$yaml" <<EOL
|
|||||||
version: '10'
|
version: '10'
|
||||||
EOL
|
EOL
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
drop dictionary if exists regexp_dict1;
|
drop dictionary if exists regexp_dict1;
|
||||||
create dictionary regexp_dict1
|
create dictionary regexp_dict1
|
||||||
(
|
(
|
||||||
@ -69,7 +69,7 @@ cat > "$yaml" <<EOL
|
|||||||
lucky: 'abcde'
|
lucky: 'abcde'
|
||||||
EOL
|
EOL
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
system reload dictionary regexp_dict1; -- { serverError 489 }
|
system reload dictionary regexp_dict1; -- { serverError 489 }
|
||||||
"
|
"
|
||||||
|
|
||||||
@ -79,7 +79,7 @@ cat > "$yaml" <<EOL
|
|||||||
version: '\1'
|
version: '\1'
|
||||||
EOL
|
EOL
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
system reload dictionary regexp_dict1; -- { serverError 318 }
|
system reload dictionary regexp_dict1; -- { serverError 318 }
|
||||||
"
|
"
|
||||||
|
|
||||||
@ -92,7 +92,7 @@ cat > "$yaml" <<EOL
|
|||||||
version: '\2.\3'
|
version: '\2.\3'
|
||||||
EOL
|
EOL
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
system reload dictionary regexp_dict1;
|
system reload dictionary regexp_dict1;
|
||||||
select dictGet('regexp_dict1', ('name', 'version'), 'Mozilla/5.0 (BB10; Touch) AppleWebKit/537.3+ (KHTML, like Gecko) Version/10.0.9.388 Mobile Safari/537.3+');
|
select dictGet('regexp_dict1', ('name', 'version'), 'Mozilla/5.0 (BB10; Touch) AppleWebKit/537.3+ (KHTML, like Gecko) Version/10.0.9.388 Mobile Safari/537.3+');
|
||||||
select dictGet('regexp_dict1', ('name', 'version'), 'Mozilla/5.0 (PlayBook; U; RIM Tablet OS 1.0.0; en-US) AppleWebKit/534.8+ (KHTML, like Gecko) Version/0.0.1 Safari/534.8+');
|
select dictGet('regexp_dict1', ('name', 'version'), 'Mozilla/5.0 (PlayBook; U; RIM Tablet OS 1.0.0; en-US) AppleWebKit/534.8+ (KHTML, like Gecko) Version/0.0.1 Safari/534.8+');
|
||||||
@ -107,7 +107,7 @@ cat > "$yaml" <<EOL
|
|||||||
col_array: '[1,2,3,-1,-2,-3]'
|
col_array: '[1,2,3,-1,-2,-3]'
|
||||||
EOL
|
EOL
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
create dictionary regexp_dict2
|
create dictionary regexp_dict2
|
||||||
(
|
(
|
||||||
regexp String,
|
regexp String,
|
||||||
@ -147,7 +147,7 @@ cat > "$yaml" <<EOL
|
|||||||
EOL
|
EOL
|
||||||
|
|
||||||
# dictGetAll
|
# dictGetAll
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
drop dictionary if exists regexp_dict3;
|
drop dictionary if exists regexp_dict3;
|
||||||
create dictionary regexp_dict3
|
create dictionary regexp_dict3
|
||||||
(
|
(
|
||||||
@ -192,7 +192,7 @@ cat > "$yaml" <<EOL
|
|||||||
tag: 'Documentation'
|
tag: 'Documentation'
|
||||||
EOL
|
EOL
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
drop dictionary if exists regexp_dict3;
|
drop dictionary if exists regexp_dict3;
|
||||||
create dictionary regexp_dict3
|
create dictionary regexp_dict3
|
||||||
(
|
(
|
||||||
@ -252,7 +252,7 @@ cat > "$yaml" <<EOL
|
|||||||
pattern: '(?-i)hello.*world'
|
pattern: '(?-i)hello.*world'
|
||||||
EOL
|
EOL
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
drop dictionary if exists regexp_dict4;
|
drop dictionary if exists regexp_dict4;
|
||||||
create dictionary regexp_dict4
|
create dictionary regexp_dict4
|
||||||
(
|
(
|
||||||
@ -291,7 +291,7 @@ select dictGetAll('regexp_dict4', 'pattern', 'HELLO WORLD');
|
|||||||
select dictGetAll('regexp_dict4', 'pattern', 'HELLO\nWORLD');
|
select dictGetAll('regexp_dict4', 'pattern', 'HELLO\nWORLD');
|
||||||
"
|
"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query="
|
$CLICKHOUSE_CLIENT --query="
|
||||||
drop dictionary regexp_dict1;
|
drop dictionary regexp_dict1;
|
||||||
drop dictionary regexp_dict2;
|
drop dictionary regexp_dict2;
|
||||||
drop dictionary regexp_dict3;
|
drop dictionary regexp_dict3;
|
||||||
|
@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# Check that if the background cleanup thread works correctly.
|
# Check that if the background cleanup thread works correctly.
|
||||||
CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}"
|
CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n --query "
|
$CLICKHOUSE_CLIENT --query "
|
||||||
DROP TABLE IF EXISTS t_async_insert_cleanup SYNC;
|
DROP TABLE IF EXISTS t_async_insert_cleanup SYNC;
|
||||||
CREATE TABLE t_async_insert_cleanup (
|
CREATE TABLE t_async_insert_cleanup (
|
||||||
KeyID UInt32
|
KeyID UInt32
|
||||||
@ -27,7 +27,7 @@ old_answer=$($CLICKHOUSE_CLIENT --query "SELECT count(*) FROM system.zookeeper W
|
|||||||
for i in {1..300}; do
|
for i in {1..300}; do
|
||||||
answer=$($CLICKHOUSE_CLIENT --query "SELECT count(*) FROM system.zookeeper WHERE path like '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_async_insert_cleanup/async_blocks%' settings allow_unrestricted_reads_from_keeper = 'true'")
|
answer=$($CLICKHOUSE_CLIENT --query "SELECT count(*) FROM system.zookeeper WHERE path like '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_async_insert_cleanup/async_blocks%' settings allow_unrestricted_reads_from_keeper = 'true'")
|
||||||
if [ $answer == '10' ]; then
|
if [ $answer == '10' ]; then
|
||||||
$CLICKHOUSE_CLIENT -n --query "DROP TABLE t_async_insert_cleanup SYNC;"
|
$CLICKHOUSE_CLIENT --query "DROP TABLE t_async_insert_cleanup SYNC;"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
@ -36,4 +36,4 @@ done
|
|||||||
$CLICKHOUSE_CLIENT --query "SELECT count(*) FROM t_async_insert_cleanup"
|
$CLICKHOUSE_CLIENT --query "SELECT count(*) FROM t_async_insert_cleanup"
|
||||||
echo $old_answer
|
echo $old_answer
|
||||||
$CLICKHOUSE_CLIENT --query "SELECT count(*) FROM system.zookeeper WHERE path like '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_async_insert_cleanup/async_blocks%' settings allow_unrestricted_reads_from_keeper = 'true'"
|
$CLICKHOUSE_CLIENT --query "SELECT count(*) FROM system.zookeeper WHERE path like '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_async_insert_cleanup/async_blocks%' settings allow_unrestricted_reads_from_keeper = 'true'"
|
||||||
$CLICKHOUSE_CLIENT -n --query "DROP TABLE t_async_insert_cleanup SYNC;"
|
$CLICKHOUSE_CLIENT --query "DROP TABLE t_async_insert_cleanup SYNC;"
|
||||||
|
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
|
|
||||||
QUERY_ID="${CLICKHOUSE_DATABASE}_test_02585_query_to_kill_id_1"
|
QUERY_ID="${CLICKHOUSE_DATABASE}_test_02585_query_to_kill_id_1"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT --query_id="$QUERY_ID" --max_rows_to_read 0 -n -q "
|
$CLICKHOUSE_CLIENT --query_id="$QUERY_ID" --max_rows_to_read 0 -q "
|
||||||
create temporary table tmp as select * from numbers(100000000);
|
create temporary table tmp as select * from numbers(100000000);
|
||||||
select * from remote('127.0.0.2', 'system.numbers_mt') where number in (select * from tmp);" &> /dev/null &
|
select * from remote('127.0.0.2', 'system.numbers_mt') where number in (select * from tmp);" &> /dev/null &
|
||||||
|
|
||||||
|
@ -13,7 +13,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
|
|
||||||
function get_query_id() { random_str 10; }
|
function get_query_id() { random_str 10; }
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
drop table if exists buf;
|
drop table if exists buf;
|
||||||
drop table if exists dist;
|
drop table if exists dist;
|
||||||
drop table if exists data;
|
drop table if exists data;
|
||||||
@ -31,7 +31,7 @@ query_id="$(get_query_id)"
|
|||||||
# test, since we care about the difference between NOW() and there should
|
# test, since we care about the difference between NOW() and there should
|
||||||
# not be any significant difference.
|
# not be any significant difference.
|
||||||
$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist"
|
$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist"
|
||||||
$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "
|
$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "
|
||||||
system flush logs;
|
system flush logs;
|
||||||
select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String};
|
select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String};
|
||||||
"
|
"
|
||||||
@ -42,25 +42,25 @@ query_id="$(get_query_id)"
|
|||||||
# this query (and all subsequent) should reuse the previous connection (at least most of the time)
|
# this query (and all subsequent) should reuse the previous connection (at least most of the time)
|
||||||
$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist"
|
$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "
|
$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "
|
||||||
system flush logs;
|
system flush logs;
|
||||||
select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String};
|
select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String};
|
||||||
"
|
"
|
||||||
|
|
||||||
echo "INSERT"
|
echo "INSERT"
|
||||||
query_id="$(get_query_id)"
|
query_id="$(get_query_id)"
|
||||||
$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -nm -q "
|
$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -m -q "
|
||||||
insert into dist_dist values (1),(2);
|
insert into dist_dist values (1),(2);
|
||||||
select * from data;
|
select * from data;
|
||||||
"
|
"
|
||||||
|
|
||||||
sleep 1
|
sleep 1
|
||||||
$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist_dist"
|
$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist_dist"
|
||||||
sleep 1
|
sleep 1
|
||||||
$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist"
|
$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist"
|
||||||
|
|
||||||
echo "CHECK"
|
echo "CHECK"
|
||||||
$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "
|
$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "
|
||||||
select * from data order by key;
|
select * from data order by key;
|
||||||
system flush logs;
|
system flush logs;
|
||||||
select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String};
|
select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String};
|
||||||
|
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
. "$CURDIR"/../shell_config.sh
|
. "$CURDIR"/../shell_config.sh
|
||||||
|
|
||||||
echo "INSERT TO S3"
|
echo "INSERT TO S3"
|
||||||
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq "
|
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q "
|
||||||
INSERT INTO TABLE FUNCTION s3('http://localhost:11111/test/profile_events.csv', 'test', 'testtest', 'CSV', 'number UInt64') SELECT number FROM numbers(1000000) SETTINGS s3_max_single_part_upload_size = 10, s3_truncate_on_insert = 1;
|
INSERT INTO TABLE FUNCTION s3('http://localhost:11111/test/profile_events.csv', 'test', 'testtest', 'CSV', 'number UInt64') SELECT number FROM numbers(1000000) SETTINGS s3_max_single_part_upload_size = 10, s3_truncate_on_insert = 1;
|
||||||
" 2>&1 | $CLICKHOUSE_LOCAL -q "
|
" 2>&1 | $CLICKHOUSE_LOCAL -q "
|
||||||
WITH '(\\w+): (\\d+)' AS pattern,
|
WITH '(\\w+): (\\d+)' AS pattern,
|
||||||
@ -30,7 +30,7 @@ SELECT * FROM (
|
|||||||
"
|
"
|
||||||
|
|
||||||
echo "CHECK WITH query_log"
|
echo "CHECK WITH query_log"
|
||||||
$CLICKHOUSE_CLIENT -nq "
|
$CLICKHOUSE_CLIENT -q "
|
||||||
SYSTEM FLUSH LOGS;
|
SYSTEM FLUSH LOGS;
|
||||||
SELECT type,
|
SELECT type,
|
||||||
'S3CreateMultipartUpload', ProfileEvents['S3CreateMultipartUpload'],
|
'S3CreateMultipartUpload', ProfileEvents['S3CreateMultipartUpload'],
|
||||||
@ -45,7 +45,7 @@ ORDER BY query_start_time DESC;
|
|||||||
"
|
"
|
||||||
|
|
||||||
echo "CREATE"
|
echo "CREATE"
|
||||||
$CLICKHOUSE_CLIENT -nq "
|
$CLICKHOUSE_CLIENT -q "
|
||||||
DROP TABLE IF EXISTS times;
|
DROP TABLE IF EXISTS times;
|
||||||
CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t
|
CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t
|
||||||
SETTINGS
|
SETTINGS
|
||||||
@ -56,29 +56,29 @@ CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t
|
|||||||
"
|
"
|
||||||
|
|
||||||
echo "INSERT"
|
echo "INSERT"
|
||||||
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq "
|
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q "
|
||||||
INSERT INTO times SELECT now() + INTERVAL 1 day SETTINGS optimize_on_insert = 0;
|
INSERT INTO times SELECT now() + INTERVAL 1 day SETTINGS optimize_on_insert = 0;
|
||||||
" 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* '
|
" 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* '
|
||||||
|
|
||||||
echo "READ"
|
echo "READ"
|
||||||
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq "
|
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q "
|
||||||
SELECT '1', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1;
|
SELECT '1', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1;
|
||||||
" 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* '
|
" 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* '
|
||||||
|
|
||||||
echo "INSERT and READ INSERT"
|
echo "INSERT and READ INSERT"
|
||||||
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq "
|
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q "
|
||||||
INSERT INTO times SELECT now() + INTERVAL 2 day SETTINGS optimize_on_insert = 0;
|
INSERT INTO times SELECT now() + INTERVAL 2 day SETTINGS optimize_on_insert = 0;
|
||||||
SELECT '2', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1;
|
SELECT '2', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1;
|
||||||
INSERT INTO times SELECT now() + INTERVAL 3 day SETTINGS optimize_on_insert = 0;
|
INSERT INTO times SELECT now() + INTERVAL 3 day SETTINGS optimize_on_insert = 0;
|
||||||
" 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* '
|
" 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* '
|
||||||
|
|
||||||
echo "DROP"
|
echo "DROP"
|
||||||
$CLICKHOUSE_CLIENT -nq "
|
$CLICKHOUSE_CLIENT -q "
|
||||||
DROP TABLE times;
|
DROP TABLE times;
|
||||||
"
|
"
|
||||||
|
|
||||||
echo "CHECK with query_log"
|
echo "CHECK with query_log"
|
||||||
$CLICKHOUSE_CLIENT -nq "
|
$CLICKHOUSE_CLIENT -q "
|
||||||
SYSTEM FLUSH LOGS;
|
SYSTEM FLUSH LOGS;
|
||||||
SELECT type,
|
SELECT type,
|
||||||
query,
|
query,
|
||||||
|
@ -21,7 +21,7 @@ wait_for_number_of_parts() {
|
|||||||
echo "$res"
|
echo "$res"
|
||||||
}
|
}
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
DROP TABLE IF EXISTS test_without_merge;
|
DROP TABLE IF EXISTS test_without_merge;
|
||||||
DROP TABLE IF EXISTS test_replicated;
|
DROP TABLE IF EXISTS test_replicated;
|
||||||
|
|
||||||
@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;"
|
|||||||
|
|
||||||
wait_for_number_of_parts 'test_without_merge' 1 10
|
wait_for_number_of_parts 'test_without_merge' 1 10
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
DROP TABLE test_without_merge;
|
DROP TABLE test_without_merge;
|
||||||
|
|
||||||
SELECT 'With merge replicated any part range';
|
SELECT 'With merge replicated any part range';
|
||||||
@ -47,7 +47,7 @@ INSERT INTO test_replicated SELECT 3;"
|
|||||||
|
|
||||||
wait_for_number_of_parts 'test_replicated' 1 100
|
wait_for_number_of_parts 'test_replicated' 1 100
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
DROP TABLE test_replicated;
|
DROP TABLE test_replicated;
|
||||||
|
|
||||||
SELECT 'With merge replicated partition only';
|
SELECT 'With merge replicated partition only';
|
||||||
@ -60,7 +60,7 @@ INSERT INTO test_replicated SELECT 3;"
|
|||||||
|
|
||||||
wait_for_number_of_parts 'test_replicated' 1 100
|
wait_for_number_of_parts 'test_replicated' 1 100
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nmq "
|
$CLICKHOUSE_CLIENT -mq "
|
||||||
SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one
|
SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one
|
||||||
SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_replicated' AND active;
|
SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_replicated' AND active;
|
||||||
|
|
||||||
|
@ -9,6 +9,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
echo "
|
echo "
|
||||||
DROP USER IF EXISTS postgresql_user;
|
DROP USER IF EXISTS postgresql_user;
|
||||||
CREATE USER postgresql_user HOST IP '127.0.0.1' IDENTIFIED WITH no_password;
|
CREATE USER postgresql_user HOST IP '127.0.0.1' IDENTIFIED WITH no_password;
|
||||||
" | $CLICKHOUSE_CLIENT -n
|
" | $CLICKHOUSE_CLIENT
|
||||||
|
|
||||||
psql --host localhost --port ${CLICKHOUSE_PORT_POSTGRESQL} ${CLICKHOUSE_DATABASE} --user postgresql_user -c "SELECT 1.23::Decimal256(70) AS test;"
|
psql --host localhost --port ${CLICKHOUSE_PORT_POSTGRESQL} ${CLICKHOUSE_DATABASE} --user postgresql_user -c "SELECT 1.23::Decimal256(70) AS test;"
|
||||||
|
@ -12,7 +12,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# too slow with this.
|
# too slow with this.
|
||||||
#
|
#
|
||||||
# Unfortunately, the test has to buffer it in memory.
|
# Unfortunately, the test has to buffer it in memory.
|
||||||
$CLICKHOUSE_CLIENT --max_memory_usage 16G -nm -q "
|
$CLICKHOUSE_CLIENT --max_memory_usage 16G -m -q "
|
||||||
INSERT INTO FUNCTION s3('http://localhost:11111/test/$CLICKHOUSE_DATABASE/test_INT_MAX.tsv', '', '', 'TSV')
|
INSERT INTO FUNCTION s3('http://localhost:11111/test/$CLICKHOUSE_DATABASE/test_INT_MAX.tsv', '', '', 'TSV')
|
||||||
SELECT repeat('a', 1024) FROM numbers((pow(2, 30) * 2) / 1024)
|
SELECT repeat('a', 1024) FROM numbers((pow(2, 30) * 2) / 1024)
|
||||||
SETTINGS s3_max_single_part_upload_size = '5Gi';
|
SETTINGS s3_max_single_part_upload_size = '5Gi';
|
||||||
|
@ -10,7 +10,7 @@ set -e
|
|||||||
NUM_REPLICAS=5
|
NUM_REPLICAS=5
|
||||||
|
|
||||||
for i in $(seq 1 $NUM_REPLICAS); do
|
for i in $(seq 1 $NUM_REPLICAS); do
|
||||||
$CLICKHOUSE_CLIENT -n -q "
|
$CLICKHOUSE_CLIENT -q "
|
||||||
DROP TABLE IF EXISTS r$i SYNC;
|
DROP TABLE IF EXISTS r$i SYNC;
|
||||||
CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x SETTINGS replicated_deduplication_window = 1, allow_remote_fs_zero_copy_replication = 1;
|
CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x SETTINGS replicated_deduplication_window = 1, allow_remote_fs_zero_copy_replication = 1;
|
||||||
"
|
"
|
||||||
|
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
|
|
||||||
for DISK in s3_disk s3_cache
|
for DISK in s3_disk s3_cache
|
||||||
do
|
do
|
||||||
${CLICKHOUSE_CLIENT} -n --query "
|
${CLICKHOUSE_CLIENT} --query "
|
||||||
DROP TABLE IF EXISTS test;
|
DROP TABLE IF EXISTS test;
|
||||||
CREATE TABLE test (id Int32, empty Array(Int32))
|
CREATE TABLE test (id Int32, empty Array(Int32))
|
||||||
ENGINE=MergeTree ORDER BY id
|
ENGINE=MergeTree ORDER BY id
|
||||||
@ -17,13 +17,13 @@ do
|
|||||||
SELECT * FROM test;
|
SELECT * FROM test;
|
||||||
"
|
"
|
||||||
|
|
||||||
${CLICKHOUSE_CLIENT} -n --query "
|
${CLICKHOUSE_CLIENT} --query "
|
||||||
BACKUP TABLE test TO Disk('backups', 'test_s3_backup');
|
BACKUP TABLE test TO Disk('backups', 'test_s3_backup');
|
||||||
DROP TABLE test;
|
DROP TABLE test;
|
||||||
RESTORE TABLE test FROM Disk('backups', 'test_s3_backup');
|
RESTORE TABLE test FROM Disk('backups', 'test_s3_backup');
|
||||||
" &>/dev/null
|
" &>/dev/null
|
||||||
|
|
||||||
${CLICKHOUSE_CLIENT} -n --query "
|
${CLICKHOUSE_CLIENT} --query "
|
||||||
SELECT * FROM test;
|
SELECT * FROM test;
|
||||||
SELECT empty FROM test;
|
SELECT empty FROM test;
|
||||||
"
|
"
|
||||||
|
@ -1,2 +1,2 @@
|
|||||||
default 127.0.0.1 9181 0 0 0 1 1 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS']
|
default 127.0.0.1 9181 0 0 0 1 1 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE']
|
||||||
zookeeper2 localhost 9181 0 0 0 1
|
zookeeper2 localhost 9181 0 0 0 1
|
||||||
|
@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CUR_DIR"/../shell_config.sh
|
. "$CUR_DIR"/../shell_config.sh
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -n -q "
|
$CLICKHOUSE_CLIENT -q "
|
||||||
CREATE TEMPORARY TABLE IF NOT EXISTS aboba
|
CREATE TEMPORARY TABLE IF NOT EXISTS aboba
|
||||||
(
|
(
|
||||||
user_id UInt32,
|
user_id UInt32,
|
||||||
|
@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CUR_DIR"/../shell_config.sh
|
. "$CUR_DIR"/../shell_config.sh
|
||||||
|
|
||||||
${CLICKHOUSE_CLIENT} -nm --query "
|
${CLICKHOUSE_CLIENT} -m --query "
|
||||||
DROP TABLE IF EXISTS test_s3;
|
DROP TABLE IF EXISTS test_s3;
|
||||||
|
|
||||||
CREATE TABLE test_s3 (a UInt64, b UInt64)
|
CREATE TABLE test_s3 (a UInt64, b UInt64)
|
||||||
@ -17,7 +17,7 @@ INSERT INTO test_s3 SELECT number, number FROM numbers(1000000);
|
|||||||
query="SELECT sum(b) FROM test_s3 WHERE a >= 100000 AND a <= 102000"
|
query="SELECT sum(b) FROM test_s3 WHERE a >= 100000 AND a <= 102000"
|
||||||
query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1)
|
query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1)
|
||||||
${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
|
${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
|
||||||
${CLICKHOUSE_CLIENT} -nm --query "
|
${CLICKHOUSE_CLIENT} -m --query "
|
||||||
SELECT
|
SELECT
|
||||||
ProfileEvents['S3ReadRequestsCount'],
|
ProfileEvents['S3ReadRequestsCount'],
|
||||||
ProfileEvents['ReadBufferFromS3Bytes'],
|
ProfileEvents['ReadBufferFromS3Bytes'],
|
||||||
|
@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CURDIR"/../shell_config.sh
|
. "$CURDIR"/../shell_config.sh
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
DROP TABLE IF EXISTS test;
|
DROP TABLE IF EXISTS test;
|
||||||
CREATE TABLE test (a Int32, b String)
|
CREATE TABLE test (a Int32, b String)
|
||||||
ENGINE = MergeTree()
|
ENGINE = MergeTree()
|
||||||
@ -22,7 +22,7 @@ INSERT INTO test SELECT number, randomString(100) FROM numbers(1000000);
|
|||||||
"
|
"
|
||||||
|
|
||||||
QUERY_ID=$RANDOM
|
QUERY_ID=$RANDOM
|
||||||
$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -nm -q "
|
$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -m -q "
|
||||||
SET enable_filesystem_cache_log = 1;
|
SET enable_filesystem_cache_log = 1;
|
||||||
SYSTEM DROP FILESYSTEM CACHE;
|
SYSTEM DROP FILESYSTEM CACHE;
|
||||||
SELECT * FROM test WHERE NOT ignore() LIMIT 1 FORMAT Null;
|
SELECT * FROM test WHERE NOT ignore() LIMIT 1 FORMAT Null;
|
||||||
@ -49,14 +49,14 @@ WHERE query_id = '$QUERY_ID' "
|
|||||||
|
|
||||||
# File segments cannot be less that 20Mi,
|
# File segments cannot be less that 20Mi,
|
||||||
# except for last file segment in a file or if file size is less.
|
# except for last file segment in a file or if file size is less.
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
SELECT count() FROM ($query)
|
SELECT count() FROM ($query)
|
||||||
WHERE file_segment_size < file_size
|
WHERE file_segment_size < file_size
|
||||||
AND end_offset + 1 != file_size
|
AND end_offset + 1 != file_size
|
||||||
AND file_segment_size < 20 * 1024 * 1024;
|
AND file_segment_size < 20 * 1024 * 1024;
|
||||||
"
|
"
|
||||||
|
|
||||||
all=$($CLICKHOUSE_CLIENT -nm -q "
|
all=$($CLICKHOUSE_CLIENT -m -q "
|
||||||
SELECT count() FROM ($query)
|
SELECT count() FROM ($query)
|
||||||
WHERE file_segment_size < file_size AND end_offset + 1 != file_size;
|
WHERE file_segment_size < file_size AND end_offset + 1 != file_size;
|
||||||
")
|
")
|
||||||
@ -68,7 +68,7 @@ else
|
|||||||
echo "FAIL"
|
echo "FAIL"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
count=$($CLICKHOUSE_CLIENT -nm -q "
|
count=$($CLICKHOUSE_CLIENT -m -q "
|
||||||
SELECT count() FROM ($query)
|
SELECT count() FROM ($query)
|
||||||
WHERE file_segment_size < file_size
|
WHERE file_segment_size < file_size
|
||||||
AND end_offset + 1 != file_size
|
AND end_offset + 1 != file_size
|
||||||
@ -87,21 +87,21 @@ FROM (SELECT * FROM ($query)) AS cache_log
|
|||||||
INNER JOIN system.filesystem_cache AS cache
|
INNER JOIN system.filesystem_cache AS cache
|
||||||
ON cache_log.cache_path = cache.cache_path "
|
ON cache_log.cache_path = cache.cache_path "
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
SELECT count() FROM ($query2)
|
SELECT count() FROM ($query2)
|
||||||
WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size
|
WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size
|
||||||
AND file_segment_range_end + 1 != file_size
|
AND file_segment_range_end + 1 != file_size
|
||||||
AND downloaded_size < 20 * 1024 * 1024;
|
AND downloaded_size < 20 * 1024 * 1024;
|
||||||
"
|
"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
SELECT count() FROM ($query2)
|
SELECT count() FROM ($query2)
|
||||||
WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size
|
WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size
|
||||||
AND file_segment_range_end + 1 != file_size
|
AND file_segment_range_end + 1 != file_size
|
||||||
AND formatReadableSize(downloaded_size) not in ('20.00 MiB', '40.00 MiB');
|
AND formatReadableSize(downloaded_size) not in ('20.00 MiB', '40.00 MiB');
|
||||||
"
|
"
|
||||||
|
|
||||||
all=$($CLICKHOUSE_CLIENT -nm -q "
|
all=$($CLICKHOUSE_CLIENT -m -q "
|
||||||
SELECT count() FROM ($query2)
|
SELECT count() FROM ($query2)
|
||||||
WHERE file_segment_size < file_size AND file_segment_range_end + 1 != file_size;
|
WHERE file_segment_size < file_size AND file_segment_range_end + 1 != file_size;
|
||||||
")
|
")
|
||||||
@ -112,7 +112,7 @@ else
|
|||||||
echo "FAIL"
|
echo "FAIL"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
count2=$($CLICKHOUSE_CLIENT -nm -q "
|
count2=$($CLICKHOUSE_CLIENT -m -q "
|
||||||
SELECT count() FROM ($query2)
|
SELECT count() FROM ($query2)
|
||||||
WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size
|
WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size
|
||||||
AND file_segment_range_end + 1 != file_size
|
AND file_segment_range_end + 1 != file_size
|
||||||
|
@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
# shellcheck source=../shell_config.sh
|
# shellcheck source=../shell_config.sh
|
||||||
. "$CUR_DIR"/../shell_config.sh
|
. "$CUR_DIR"/../shell_config.sh
|
||||||
|
|
||||||
${CLICKHOUSE_CLIENT} -nm --query "
|
${CLICKHOUSE_CLIENT} -m --query "
|
||||||
DROP TABLE IF EXISTS test_s3;
|
DROP TABLE IF EXISTS test_s3;
|
||||||
|
|
||||||
CREATE TABLE test_s3 (a UInt64, b UInt64)
|
CREATE TABLE test_s3 (a UInt64, b UInt64)
|
||||||
@ -25,7 +25,7 @@ do
|
|||||||
query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1)
|
query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1)
|
||||||
${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
|
${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
|
||||||
|
|
||||||
RES=$(${CLICKHOUSE_CLIENT} -nm --query "
|
RES=$(${CLICKHOUSE_CLIENT} -m --query "
|
||||||
SELECT ProfileEvents['DiskConnectionsPreserved'] > 0
|
SELECT ProfileEvents['DiskConnectionsPreserved'] > 0
|
||||||
FROM system.query_log
|
FROM system.query_log
|
||||||
WHERE type = 'QueryFinish'
|
WHERE type = 'QueryFinish'
|
||||||
@ -41,7 +41,7 @@ done
|
|||||||
|
|
||||||
while true
|
while true
|
||||||
do
|
do
|
||||||
query_id=$(${CLICKHOUSE_CLIENT} -nq "
|
query_id=$(${CLICKHOUSE_CLIENT} -q "
|
||||||
create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n;
|
create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n;
|
||||||
set insert_keeper_fault_injection_probability=0;
|
set insert_keeper_fault_injection_probability=0;
|
||||||
insert into mut values (1, 2, 3), (10, 20, 30);
|
insert into mut values (1, 2, 3), (10, 20, 30);
|
||||||
@ -60,7 +60,7 @@ do
|
|||||||
) limit 1 settings max_threads=1;
|
) limit 1 settings max_threads=1;
|
||||||
" 2>&1)
|
" 2>&1)
|
||||||
${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
|
${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
|
||||||
RES=$(${CLICKHOUSE_CLIENT} -nm --query "
|
RES=$(${CLICKHOUSE_CLIENT} -m --query "
|
||||||
SELECT ProfileEvents['StorageConnectionsPreserved'] > 0
|
SELECT ProfileEvents['StorageConnectionsPreserved'] > 0
|
||||||
FROM system.query_log
|
FROM system.query_log
|
||||||
WHERE type = 'QueryFinish'
|
WHERE type = 'QueryFinish'
|
||||||
|
@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
drop table if exists data;
|
drop table if exists data;
|
||||||
create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk';
|
create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk';
|
||||||
insert into data select * from numbers(10);
|
insert into data select * from numbers(10);
|
||||||
@ -16,28 +16,28 @@ $CLICKHOUSE_CLIENT -nm -q "
|
|||||||
|
|
||||||
query_id=$(random_str 10)
|
query_id=$(random_str 10)
|
||||||
$CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true"
|
$CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true"
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
SYSTEM FLUSH LOGS;
|
SYSTEM FLUSH LOGS;
|
||||||
SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id'
|
SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id'
|
||||||
"
|
"
|
||||||
|
|
||||||
query_id=$(random_str 10)
|
query_id=$(random_str 10)
|
||||||
$CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false"
|
$CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false"
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
SYSTEM FLUSH LOGS;
|
SYSTEM FLUSH LOGS;
|
||||||
SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id'
|
SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id'
|
||||||
"
|
"
|
||||||
|
|
||||||
query_id=$(random_str 10)
|
query_id=$(random_str 10)
|
||||||
$CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true"
|
$CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true"
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
SYSTEM FLUSH LOGS;
|
SYSTEM FLUSH LOGS;
|
||||||
SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id'
|
SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id'
|
||||||
"
|
"
|
||||||
|
|
||||||
query_id=$(random_str 10)
|
query_id=$(random_str 10)
|
||||||
$CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_no_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false"
|
$CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_no_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false"
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
SYSTEM FLUSH LOGS;
|
SYSTEM FLUSH LOGS;
|
||||||
SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id'
|
SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id'
|
||||||
"
|
"
|
||||||
|
@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm -q "
|
$CLICKHOUSE_CLIENT -m -q "
|
||||||
drop table if exists data;
|
drop table if exists data;
|
||||||
create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk';
|
create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk';
|
||||||
insert into data select * from numbers(10);
|
insert into data select * from numbers(10);
|
||||||
|
@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|||||||
. "$CUR_DIR"/../shell_config.sh
|
. "$CUR_DIR"/../shell_config.sh
|
||||||
|
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm --query """
|
$CLICKHOUSE_CLIENT -m --query """
|
||||||
DROP TABLE IF EXISTS test;
|
DROP TABLE IF EXISTS test;
|
||||||
CREATE TABLE test (a Int32, b String)
|
CREATE TABLE test (a Int32, b String)
|
||||||
ENGINE = MergeTree() ORDER BY tuple()
|
ENGINE = MergeTree() ORDER BY tuple()
|
||||||
@ -17,17 +17,17 @@ SETTINGS disk = disk(name = 's3_disk', type = cache, max_size = '100Ki', path =
|
|||||||
|
|
||||||
disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}"
|
disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}"
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm --query """
|
$CLICKHOUSE_CLIENT -m --query """
|
||||||
SELECT count() FROM system.disks WHERE name = '$disk_name'
|
SELECT count() FROM system.disks WHERE name = '$disk_name'
|
||||||
"""
|
"""
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm --query """
|
$CLICKHOUSE_CLIENT -m --query """
|
||||||
DROP TABLE IF EXISTS test;
|
DROP TABLE IF EXISTS test;
|
||||||
CREATE TABLE test (a Int32, b String)
|
CREATE TABLE test (a Int32, b String)
|
||||||
ENGINE = MergeTree() ORDER BY tuple()
|
ENGINE = MergeTree() ORDER BY tuple()
|
||||||
SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = ${CLICKHOUSE_TEST_UNIQUE_NAME}, disk = s3_disk);
|
SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = ${CLICKHOUSE_TEST_UNIQUE_NAME}, disk = s3_disk);
|
||||||
"""
|
"""
|
||||||
|
|
||||||
$CLICKHOUSE_CLIENT -nm --query """
|
$CLICKHOUSE_CLIENT -m --query """
|
||||||
SELECT count() FROM system.disks WHERE name = '$disk_name'
|
SELECT count() FROM system.disks WHERE name = '$disk_name'
|
||||||
"""
|
"""
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user