thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into json-data-type

Browse Source
This commit is contained in:
Kruglov Pavel 2024-08-12 23:32:35 +02:00 committed by GitHub
commit 01b2e67f2e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
129 changed files with 1310 additions and 659 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
docker/test/stateless/run.sh
View File

@ -54,8 +54,6 @@ source /utils.lib
/usr/share/clickhouse-test/config/install.sh
./setup_minio.sh stateless
./mc admin trace clickminio > /test_output/minio.log &
MC_ADMIN_PID=$!
./setup_hdfs_minicluster.sh
@ -176,6 +174,55 @@ done
setup_logs_replication
attach_gdb_to_clickhouse
# create tables for minio log webhooks
clickhouse-client --query "CREATE TABLE minio_audit_logs
(
log String,
event_time DateTime64(9) MATERIALIZED parseDateTime64BestEffortOrZero(trim(BOTH '\"' FROM JSONExtractRaw(log, 'time')), 9, 'UTC')
)
ENGINE = MergeTree
ORDER BY tuple()"
clickhouse-client --query "CREATE TABLE minio_server_logs
(
log String,
event_time DateTime64(9) MATERIALIZED parseDateTime64BestEffortOrZero(trim(BOTH '\"' FROM JSONExtractRaw(log, 'time')), 9, 'UTC')
)
ENGINE = MergeTree
ORDER BY tuple()"
# create minio log webhooks for both audit and server logs
# use async inserts to avoid creating too many parts
./mc admin config set clickminio logger_webhook:ch_server_webhook endpoint="http://localhost:8123/?async_insert=1&wait_for_async_insert=0&async_insert_busy_timeout_min_ms=5000&async_insert_busy_timeout_max_ms=5000&async_insert_max_query_number=1000&async_insert_max_data_size=10485760&query=INSERT%20INTO%20minio_server_logs%20FORMAT%20LineAsString" queue_size=1000000 batch_size=500
./mc admin config set clickminio audit_webhook:ch_audit_webhook endpoint="http://localhost:8123/?async_insert=1&wait_for_async_insert=0&async_insert_busy_timeout_min_ms=5000&async_insert_busy_timeout_max_ms=5000&async_insert_max_query_number=1000&async_insert_max_data_size=10485760&query=INSERT%20INTO%20minio_audit_logs%20FORMAT%20LineAsString" queue_size=1000000 batch_size=500
max_retries=100
retry=1
while [ $retry -le $max_retries ]; do
echo "clickminio restart attempt $retry:"
output=$(./mc admin service restart clickminio --wait --json 2>&1 | jq -r .status)
echo "Output of restart status: $output"
expected_output="success
success"
if [ "$output" = "$expected_output" ]; then
echo "Restarted clickminio successfully."
break
fi
sleep 1
retry=$((retry + 1))
done
if [ $retry -gt $max_retries ]; then
echo "Failed to restart clickminio after $max_retries attempts."
fi
./mc admin trace clickminio > /test_output/minio.log &
MC_ADMIN_PID=$!
function fn_exists() {
declare -F "$1" > /dev/null;
}
@ -339,6 +386,14 @@ do
fi
done
# collect minio audit and server logs
# wait for minio to flush its batch if it has any
sleep 1
clickhouse-client -q "SYSTEM FLUSH ASYNC INSERT QUEUE"
clickhouse-client -q "SELECT log FROM minio_audit_logs ORDER BY event_time INTO OUTFILE '/test_output/minio_audit_logs.jsonl.zst' FORMAT JSONEachRow"
clickhouse-client -q "SELECT log FROM minio_server_logs ORDER BY event_time INTO OUTFILE '/test_output/minio_server_logs.jsonl.zst' FORMAT JSONEachRow"
# Stop server so we can safely read data with clickhouse-local.
# Why do we read data with clickhouse-local?
# Because it's the simplest way to read it when server has crashed.

4
docs/en/development/tests.md
View File

@ -14,7 +14,7 @@ Each functional test sends one or multiple queries to the running ClickHouse ser
Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from ClickHouse and it is available to general public.
Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client --multiquery`. `.sh` test is a script that is run by itself. SQL tests are generally preferable to `.sh` tests. You should use `.sh` tests only when you have to test some feature that cannot be exercised from pure SQL, such as piping some input data into `clickhouse-client` or testing `clickhouse-local`.
Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client`. `.sh` test is a script that is run by itself. SQL tests are generally preferable to `.sh` tests. You should use `.sh` tests only when you have to test some feature that cannot be exercised from pure SQL, such as piping some input data into `clickhouse-client` or testing `clickhouse-local`.
:::note
A common mistake when testing data types `DateTime` and `DateTime64` is assuming that the server uses a specific time zone (e.g. "UTC"). This is not the case, time zones in CI test runs
@ -38,7 +38,7 @@ For more options, see `tests/clickhouse-test --help`. You can simply run all tes
### Adding a New Test
To add new test, create a `.sql` or `.sh` file in `queries/0_stateless` directory, check it manually and then generate `.reference` file in the following way: `clickhouse-client --multiquery < 00000_test.sql > 00000_test.reference` or `./00000_test.sh > ./00000_test.reference`.
To add new test, create a `.sql` or `.sh` file in `queries/0_stateless` directory, check it manually and then generate `.reference` file in the following way: `clickhouse-client < 00000_test.sql > 00000_test.reference` or `./00000_test.sh > ./00000_test.reference`.
Tests should use (create, drop, etc) only tables in `test` database that is assumed to be created beforehand; also tests can use temporary tables.

16
docs/en/sql-reference/statements/alter/statistics.md
View File

@ -8,26 +8,28 @@ sidebar_label: STATISTICS
The following operations are available:
- `ALTER TABLE [db].table ADD STATISTICS (columns list) TYPE (type list)` - Adds statistic description to tables metadata.
- `ALTER TABLE [db].table ADD STATISTICS [IF NOT EXISTS] (column list) TYPE (type list)` - Adds statistic description to tables metadata.
- `ALTER TABLE [db].table MODIFY STATISTICS (columns list) TYPE (type list)` - Modifies statistic description to tables metadata.
- `ALTER TABLE [db].table MODIFY STATISTICS (column list) TYPE (type list)` - Modifies statistic description to tables metadata.
- `ALTER TABLE [db].table DROP STATISTICS (columns list)` - Removes statistics from the metadata of the specified columns and deletes all statistics objects in all parts for the specified columns.
- `ALTER TABLE [db].table DROP STATISTICS [IF EXISTS] (column list)` - Removes statistics from the metadata of the specified columns and deletes all statistics objects in all parts for the specified columns.
- `ALTER TABLE [db].table CLEAR STATISTICS (columns list)` - Deletes all statistics objects in all parts for the specified columns. Statistics objects can be rebuild using `ALTER TABLE MATERIALIZE STATISTICS`.
- `ALTER TABLE [db].table CLEAR STATISTICS [IF EXISTS] (column list)` - Deletes all statistics objects in all parts for the specified columns. Statistics objects can be rebuild using `ALTER TABLE MATERIALIZE STATISTICS`.
- `ALTER TABLE [db.]table MATERIALIZE STATISTICS (columns list)` - Rebuilds the statistic for columns. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
- `ALTER TABLE [db.]table MATERIALIZE STATISTICS [IF EXISTS] (column list)` - Rebuilds the statistic for columns. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
The first two commands are lightweight in a sense that they only change metadata or remove files.
Also, they are replicated, syncing statistics metadata via ZooKeeper.
There is an example adding two statistics types to two columns:
## Example:
Adding two statistics types to two columns:
```
ALTER TABLE t1 MODIFY STATISTICS c, d TYPE TDigest, Uniq;
```
:::note
Statistic manipulation is supported only for tables with [`*MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) engine (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) variants).
Statistic are supported only for [`*MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) engine tables (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) variants).
:::

16
docs/zh/operations/external-authenticators/kerberos.md
View File

@ -23,30 +23,30 @@ slug: /zh/operations/external-authenticators/kerberos
示例 (进入 `config.xml`):
```xml
<yandex>
<clickhouse>
<!- ... -->
<kerberos />
</yandex>
</clickhouse>
```
主体规范:
```xml
<yandex>
<clickhouse>
<!- ... -->
<kerberos>
<principal>HTTP/clickhouse.example.com@EXAMPLE.COM</principal>
</kerberos>
</yandex>
</clickhouse>
```
按领域过滤:
```xml
<yandex>
<clickhouse>
<!- ... -->
<kerberos>
<realm>EXAMPLE.COM</realm>
</kerberos>
</yandex>
</clickhouse>
```
!!! warning "注意"
@ -74,7 +74,7 @@ Kerberos主体名称格式通常遵循以下模式:
示例 (进入 `users.xml`):
```
<yandex>
<clickhouse>
<!- ... -->
<users>
<!- ... -->
@ -85,7 +85,7 @@ Kerberos主体名称格式通常遵循以下模式:
</kerberos>
</my_user>
</users>
</yandex>
</clickhouse>
```
!!! warning "警告"

1
programs/server/config.d/transactions.xml Symbolic link
View File

@ -0,0 +1 @@
../../../tests/config/config.d/transactions.xml

16
src/Columns/ColumnArray.cpp
View File

@ -457,6 +457,22 @@ size_t ColumnArray::capacity() const
return getOffsets().capacity();
}
void ColumnArray::prepareForSquashing(const Columns & source_columns)
{
size_t new_size = size();
Columns source_data_columns;
source_data_columns.reserve(source_columns.size());
for (const auto & source_column : source_columns)
{
const auto & source_array_column = assert_cast<const ColumnArray &>(*source_column);
new_size += source_array_column.size();
source_data_columns.push_back(source_array_column.getDataPtr());
}
getOffsets().reserve_exact(new_size);
data->prepareForSquashing(source_data_columns);
}
void ColumnArray::shrinkToFit()
{
getOffsets().shrink_to_fit();

1
src/Columns/ColumnArray.h
View File

@ -119,6 +119,7 @@ public:
size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_ranges) const override;
void reserve(size_t n) override;
size_t capacity() const override;
void prepareForSquashing(const Columns & source_columns) override;
void shrinkToFit() override;
void ensureOwnership() override;
size_t byteSize() const override;

110
src/Columns/ColumnDynamic.cpp
View File

@ -655,6 +655,116 @@ ColumnPtr ColumnDynamic::compress() const
});
}
void ColumnDynamic::prepareForSquashing(const Columns & source_columns)
{
if (source_columns.empty())
return;
/// Internal variants of source dynamic columns may differ.
/// We want to preallocate memory for all variants we will have after squashing.
/// It may happen that the total number of variants in source columns will
/// exceed the limit, in this case we will choose the most frequent variants.
/// First, preallocate memory for variant discriminators and offsets.
size_t new_size = size();
for (const auto & source_column : source_columns)
new_size += source_column->size();
auto & variant_col = getVariantColumn();
variant_col.getLocalDiscriminators().reserve_exact(new_size);
variant_col.getOffsets().reserve_exact(new_size);
/// Second, collect all variants and their total sizes.
std::unordered_map<String, size_t> total_variant_sizes;
DataTypes all_variants;
auto add_variants = [&](const ColumnDynamic & source_dynamic)
{
const auto & source_variant_column = source_dynamic.getVariantColumn();
const auto & source_variant_info = source_dynamic.getVariantInfo();
const auto & source_variants = assert_cast<const DataTypeVariant &>(*source_variant_info.variant_type).getVariants();
for (size_t i = 0; i != source_variants.size(); ++i)
{
const auto & variant_name = source_variant_info.variant_names[i];
auto it = total_variant_sizes.find(variant_name);
/// Add this variant to the list of all variants if we didn't see it yet.
if (it == total_variant_sizes.end())
{
all_variants.push_back(source_variants[i]);
it = total_variant_sizes.emplace(variant_name, 0).first;
}
it->second += source_variant_column.getVariantByGlobalDiscriminator(i).size();
}
};
for (const auto & source_column : source_columns)
add_variants(assert_cast<const ColumnDynamic &>(*source_column));
/// Add variants from this dynamic column.
add_variants(*this);
DataTypePtr result_variant_type;
/// Check if the number of all variants exceeds the limit.
if (all_variants.size() > max_dynamic_types || (all_variants.size() == max_dynamic_types && !total_variant_sizes.contains("String")))
{
/// We want to keep the most frequent variants in the resulting dynamic column.
DataTypes result_variants;
result_variants.reserve(max_dynamic_types);
/// Add variants from current variant column as we will not rewrite it.
for (const auto & variant : assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariants())
result_variants.push_back(variant);
/// Add String variant in advance (if we didn't add it yet) as we must have it across variants when we reach the limit.
if (!variant_info.variant_name_to_discriminator.contains("String"))
result_variants.push_back(std::make_shared<DataTypeString>());
/// Create list of remaining variants with their sizes and sort it.
std::vector<std::pair<size_t, DataTypePtr>> variants_with_sizes;
variants_with_sizes.reserve(all_variants.size() - variant_info.variant_names.size());
for (const auto & variant : all_variants)
{
/// Add variant to the list only of we didn't add it yet.
auto variant_name = variant->getName();
if (variant_name != "String" && !variant_info.variant_name_to_discriminator.contains(variant_name))
variants_with_sizes.emplace_back(total_variant_sizes[variant->getName()], variant);
}
std::sort(variants_with_sizes.begin(), variants_with_sizes.end(), std::greater());
/// Add the most frequent variants until we reach max_dynamic_types.
size_t num_new_variants = max_dynamic_types - result_variants.size();
for (size_t i = 0; i != num_new_variants; ++i)
result_variants.push_back(variants_with_sizes[i].second);
result_variant_type = std::make_shared<DataTypeVariant>(result_variants);
}
else
{
result_variant_type = std::make_shared<DataTypeVariant>(all_variants);
}
if (!result_variant_type->equals(*variant_info.variant_type))
updateVariantInfoAndExpandVariantColumn(result_variant_type);
/// Now current dynamic column has all resulting variants and we can call
/// prepareForSquashing on them to preallocate the memory.
for (size_t i = 0; i != variant_info.variant_names.size(); ++i)
{
Columns source_variant_columns;
source_variant_columns.reserve(source_columns.size());
for (const auto & source_column : source_columns)
{
const auto & source_dynamic_column = assert_cast<const ColumnDynamic &>(*source_column);
const auto & source_variant_info = source_dynamic_column.getVariantInfo();
/// Try to find this variant in the current source column.
auto it = source_variant_info.variant_name_to_discriminator.find(variant_info.variant_names[i]);
if (it != source_variant_info.variant_name_to_discriminator.end())
source_variant_columns.push_back(source_dynamic_column.getVariantColumn().getVariantPtrByGlobalDiscriminator(it->second));
}
variant_col.getVariantByGlobalDiscriminator(i).prepareForSquashing(source_variant_columns);
}
}
void ColumnDynamic::takeDynamicStructureFromSourceColumns(const Columns & source_columns)
{
if (!empty())

2
src/Columns/ColumnDynamic.h
View File

@ -258,6 +258,8 @@ public:
return variant_column_ptr->capacity();
}
void prepareForSquashing(const Columns & source_columns) override;
void ensureOwnership() override
{
variant_column_ptr->ensureOwnership();

9
src/Columns/ColumnMap.cpp
View File

@ -254,6 +254,15 @@ size_t ColumnMap::capacity() const
return nested->capacity();
}
void ColumnMap::prepareForSquashing(const Columns & source_columns)
{
Columns nested_source_columns;
nested_source_columns.reserve(source_columns.size());
for (const auto & source_column : source_columns)
nested_source_columns.push_back(assert_cast<const ColumnMap &>(*source_column).getNestedColumnPtr());
nested->prepareForSquashing(nested_source_columns);
}
void ColumnMap::shrinkToFit()
{
nested->shrinkToFit();

1
src/Columns/ColumnMap.h
View File

@ -95,6 +95,7 @@ public:
size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const override;
void reserve(size_t n) override;
size_t capacity() const override;
void prepareForSquashing(const Columns & source_columns) override;
void shrinkToFit() override;
void ensureOwnership() override;
size_t byteSize() const override;

16
src/Columns/ColumnNullable.cpp
View File

@ -711,6 +711,22 @@ size_t ColumnNullable::capacity() const
return getNullMapData().capacity();
}
void ColumnNullable::prepareForSquashing(const Columns & source_columns)
{
size_t new_size = size();
Columns nested_source_columns;
nested_source_columns.reserve(source_columns.size());
for (const auto & source_column : source_columns)
{
const auto & source_nullable_column = assert_cast<const ColumnNullable &>(*source_column);
new_size += source_nullable_column.size();
nested_source_columns.push_back(source_nullable_column.getNestedColumnPtr());
}
nested_column->prepareForSquashing(nested_source_columns);
getNullMapData().reserve(new_size);
}
void ColumnNullable::shrinkToFit()
{
getNestedColumn().shrinkToFit();

1
src/Columns/ColumnNullable.h
View File

@ -126,6 +126,7 @@ public:
size_t estimateCardinalityInPermutedRange(const Permutation & permutation, const EqualRange & equal_range) const override;
void reserve(size_t n) override;
size_t capacity() const override;
void prepareForSquashing(const Columns & source_columns) override;
void shrinkToFit() override;
void ensureOwnership() override;
size_t byteSize() const override;

15
src/Columns/ColumnString.cpp
View File

@ -562,6 +562,21 @@ size_t ColumnString::capacity() const
return offsets.capacity();
}
void ColumnString::prepareForSquashing(const Columns & source_columns)
{
size_t new_size = size();
size_t new_chars_size = chars.size();
for (const auto & source_column : source_columns)
{
const auto & source_string_column = assert_cast<const ColumnString &>(*source_column);
new_size += source_string_column.size();
new_chars_size += source_string_column.chars.size();
}
offsets.reserve_exact(new_size);
chars.reserve_exact(new_chars_size);
}
void ColumnString::shrinkToFit()
{
chars.shrink_to_fit();

1
src/Columns/ColumnString.h
View File

@ -284,6 +284,7 @@ public:
void reserve(size_t n) override;
size_t capacity() const override;
void prepareForSquashing(const Columns & source_columns) override;
void shrinkToFit() override;
void getExtremes(Field & min, Field & max) const override;

13
src/Columns/ColumnTuple.cpp
View File

@ -603,6 +603,19 @@ size_t ColumnTuple::capacity() const
return getColumn(0).capacity();
}
void ColumnTuple::prepareForSquashing(const Columns & source_columns)
{
const size_t tuple_size = columns.size();
for (size_t i = 0; i < tuple_size; ++i)
{
Columns nested_columns;
nested_columns.reserve(source_columns.size());
for (const auto & source_column : source_columns)
nested_columns.push_back(assert_cast<const ColumnTuple &>(*source_column).getColumnPtr(i));
getColumn(i).prepareForSquashing(nested_columns);
}
}
void ColumnTuple::shrinkToFit()
{
const size_t tuple_size = columns.size();

1
src/Columns/ColumnTuple.h
View File

@ -111,6 +111,7 @@ public:
size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_ranges) const override;
void reserve(size_t n) override;
size_t capacity() const override;
void prepareForSquashing(const Columns & source_columns) override;
void shrinkToFit() override;
void ensureOwnership() override;
size_t byteSize() const override;

21
src/Columns/ColumnVariant.cpp
View File

@ -1247,8 +1247,25 @@ void ColumnVariant::updatePermutation(IColumn::PermutationSortDirection directio
void ColumnVariant::reserve(size_t n)
{
local_discriminators->reserve(n);
offsets->reserve(n);
getLocalDiscriminators().reserve_exact(n);
getOffsets().reserve_exact(n);
}
void ColumnVariant::prepareForSquashing(const Columns & source_columns)
{
size_t new_size = size();
for (const auto & source_column : source_columns)
new_size += source_column->size();
reserve(new_size);
for (size_t i = 0; i != variants.size(); ++i)
{
Columns source_variant_columns;
source_variant_columns.reserve(source_columns.size());
for (const auto & source_column : source_columns)
source_variant_columns.push_back(assert_cast<const ColumnVariant &>(*source_column).getVariantPtrByGlobalDiscriminator(i));
getVariantByGlobalDiscriminator(i).prepareForSquashing(source_variant_columns);
}
}
size_t ColumnVariant::capacity() const

1
src/Columns/ColumnVariant.h
View File

@ -238,6 +238,7 @@ public:
void reserve(size_t n) override;
size_t capacity() const override;
void prepareForSquashing(const Columns & source_columns) override;
void ensureOwnership() override;
size_t byteSize() const override;
size_t byteSizeAt(size_t n) const override;

9
src/Columns/IColumn.h
View File

@ -478,6 +478,15 @@ public:
/// Returns the number of elements allocated in reserve.
virtual size_t capacity() const { return size(); }
/// Reserve memory before squashing all specified source columns into this column.
virtual void prepareForSquashing(const std::vector<Ptr> & source_columns)
{
size_t new_size = size();
for (const auto & source_column : source_columns)
new_size += source_column->size();
reserve(new_size);
}
/// Requests the removal of unused capacity.
/// It is a non-binding request to reduce the capacity of the underlying container to its size.
virtual void shrinkToFit() {}

27
src/Interpreters/Squashing.cpp
View File

@ -5,7 +5,6 @@
#include <Common/CurrentThread.h>
#include <base/defines.h>
namespace DB
{
@ -125,20 +124,32 @@ Chunk Squashing::squash(std::vector<Chunk> && input_chunks, Chunk::ChunkInfoColl
{
auto & first_chunk = input_chunks[0];
Columns columns = first_chunk.detachColumns();
mutable_columns.reserve(columns.size());
for (auto & column : columns)
{
mutable_columns.push_back(IColumn::mutate(std::move(column)));
mutable_columns.back()->reserve(rows);
}
}
size_t num_columns = mutable_columns.size();
/// Collect the list of source columns for each column.
std::vector<Columns> source_columns_list(num_columns, Columns{});
for (size_t i = 0; i != num_columns; ++i)
source_columns_list[i].reserve(input_chunks.size() - 1);
for (size_t i = 1; i < input_chunks.size(); ++i) // We've already processed the first chunk above
{
Columns columns = input_chunks[i].detachColumns();
for (size_t j = 0, size = mutable_columns.size(); j < size; ++j)
auto columns = input_chunks[i].detachColumns();
for (size_t j = 0; j != num_columns; ++j)
source_columns_list[j].emplace_back(std::move(columns[j]));
}
for (size_t i = 0; i != num_columns; ++i)
{
/// We know all the data we will insert in advance and can make all necessary pre-allocations.
mutable_columns[i]->prepareForSquashing(source_columns_list[i]);
for (auto & source_column : source_columns_list[i])
{
const auto source_column = columns[j];
mutable_columns[j]->insertRangeFrom(*source_column, 0, source_column->size());
auto column = std::move(source_column);
mutable_columns[i]->insertRangeFrom(*column, 0, column->size());
}
}

2
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -3517,7 +3517,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
const auto & new_column = new_metadata.getColumns().get(command.column_name);
if (!old_column.type->equals(*new_column.type))
throw Exception(ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN,
"ALTER types of column {} with statistics is not not safe "
"ALTER types of column {} with statistics is not safe "
"because it can change the representation of statistics",
backQuoteIfNeed(command.column_name));
}

4
tests/config/config.d/transactions.xml
View File

@ -1,4 +1,4 @@
<yandex>
<clickhouse>
<allow_experimental_transactions>42</allow_experimental_transactions>
<merge_tree>
@ -18,4 +18,4 @@
<!-- Fault injection after commit should not affect tests, because default waiting mode is WAIT_UNKNOWN -->
<fault_probability_after_commit>0.01</fault_probability_after_commit>
</transaction_log>
</yandex>
</clickhouse>

4
tests/integration/test_distributed_type_object/configs/remote_servers.xml
View File

@ -1,4 +1,4 @@
<yandex>
<clickhouse>
<remote_servers>
<test_cluster>
<shard>
@ -15,4 +15,4 @@
</shard>
</test_cluster>
</remote_servers>
</yandex>
</clickhouse>

4
tests/integration/test_host_regexp_hosts_file_resolution/configs/host_regexp.xml
View File

@ -1,4 +1,4 @@
<yandex>
<clickhouse>
<users>
<test_dns>
<password/>
@ -8,4 +8,4 @@
<profile>default</profile>
</test_dns>
</users>
</yandex>
</clickhouse>

4
tests/integration/test_host_regexp_hosts_file_resolution/configs/listen_host.xml
View File

@ -1,5 +1,5 @@
<yandex>
<clickhouse>
<listen_host>::</listen_host>
<listen_host>0.0.0.0</listen_host>
<listen_try>1</listen_try>
</yandex>
</clickhouse>

4
tests/integration/test_jbod_ha/configs/config.d/storage_configuration.xml
View File

@ -1,4 +1,4 @@
<yandex>
<clickhouse>
<local_disk_check_period_ms>1000</local_disk_check_period_ms>
<storage_configuration>
<disks>
@ -27,4 +27,4 @@
</jbod>
</policies>
</storage_configuration>
</yandex>
</clickhouse>

4
tests/integration/test_server_reload/configs/default_passwd.xml
View File

@ -1,4 +1,4 @@
<yandex>
<clickhouse>
<profiles>
<default>
</default>
@ -9,4 +9,4 @@
<password>123</password>
</postgresql>
</users>
</yandex>
</clickhouse>

4
tests/integration/test_server_reload/configs/overrides_from_zk.xml
View File

@ -1,4 +1,4 @@
<yandex>
<clickhouse>
<listen_host remove="remove"/>
<include from_zk="/clickhouse/listen_hosts" />
<tcp_port from_zk="/clickhouse/ports/tcp" replace="replace" />
@ -7,4 +7,4 @@
<postgresql_port from_zk="/clickhouse/ports/postgresql" replace="replace" />
<grpc_port from_zk="/clickhouse/ports/grpc" replace="replace" />
<http_handlers from_zk="/clickhouse/http_handlers" replace="replace" />
</yandex>
</clickhouse>

26
tests/integration/test_storage_kafka/test.py
View File

@ -1019,7 +1019,7 @@ def test_kafka_formats(kafka_cluster, create_query_generator):
DROP TABLE IF EXISTS test.kafka_{format_name}_mv;
CREATE MATERIALIZED VIEW test.kafka_{format_name}_mv Engine=Log AS
CREATE MATERIALIZED VIEW test.kafka_{format_name}_mv ENGINE=MergeTree ORDER BY tuple() AS
SELECT *, _topic, _partition, _offset FROM test.kafka_{format_name};
""".format(
topic_name=topic_name,
@ -2460,7 +2460,7 @@ def test_kafka_commit_on_block_write(kafka_cluster, create_query_generator):
(generate_old_create_table_query, "kafka.*Committed offset 2.*virt2_[01]"),
(
generate_new_create_table_query,
r"kafka.*Saved offset 2[0-9]* for topic-partition \[virt2_[01]:[0-9]+",
r"kafka.*Saved offset 2 for topic-partition \[virt2_[01]:[0-9]+",
),
],
)
@ -2494,7 +2494,7 @@ def test_kafka_virtual_columns2(kafka_cluster, create_query_generator, log_line)
f"""
{create_query};
CREATE MATERIALIZED VIEW test.view Engine=Log AS
CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY tuple() AS
SELECT value, _key, _topic, _partition, _offset, toUnixTimestamp(_timestamp), toUnixTimestamp64Milli(_timestamp_ms), _headers.name, _headers.value FROM test.kafka;
"""
)
@ -2729,7 +2729,7 @@ def test_kafka_produce_key_timestamp(kafka_cluster, create_query_generator, log_
DROP TABLE IF EXISTS test.consumer;
{writer_create_query};
{reader_create_query};
CREATE MATERIALIZED VIEW test.view Engine=Log AS
CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY tuple() AS
SELECT key, value, inserted_key, toUnixTimestamp(inserted_timestamp), _key, _topic, _partition, _offset, toUnixTimestamp(_timestamp) FROM test.kafka;
"""
)
@ -2865,7 +2865,7 @@ def test_kafka_produce_consume_avro(kafka_cluster, create_query_generator):
{writer_create_query};
{reader_create_query};
CREATE MATERIALIZED VIEW test.view Engine=Log AS
CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY tuple() AS
SELECT key, value FROM test.kafka;
"""
)
@ -3537,7 +3537,7 @@ def test_bad_reschedule(kafka_cluster, create_query_generator):
f"""
{create_query};
CREATE MATERIALIZED VIEW test.destination Engine=Log AS
CREATE MATERIALIZED VIEW test.destination ENGINE=MergeTree ORDER BY tuple() AS
SELECT
key,
now() as consume_ts,
@ -3745,7 +3745,7 @@ def test_kafka_unavailable(kafka_cluster, create_query_generator, do_direct_read
f"""
{create_query};
CREATE MATERIALIZED VIEW test.destination_unavailable Engine=Log AS
CREATE MATERIALIZED VIEW test.destination_unavailable ENGINE=MergeTree ORDER BY tuple() AS
SELECT
key,
now() as consume_ts,
@ -4267,12 +4267,12 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator
{create_query};
DROP TABLE IF EXISTS test.kafka_data_{format_name}_mv;
CREATE MATERIALIZED VIEW test.kafka_data_{format_name}_mv Engine=Log AS
CREATE MATERIALIZED VIEW test.kafka_data_{format_name}_mv ENGINE=MergeTree ORDER BY tuple() AS
SELECT *, _topic, _partition, _offset FROM test.kafka_{format_name}
WHERE length(_error) = 0;
DROP TABLE IF EXISTS test.kafka_errors_{format_name}_mv;
CREATE MATERIALIZED VIEW test.kafka_errors_{format_name}_mv Engine=Log AS
CREATE MATERIALIZED VIEW test.kafka_errors_{format_name}_mv ENGINE=MergeTree ORDER BY tuple() AS
SELECT {raw_message} as raw_message, _error as error, _topic as topic, _partition as partition, _offset as offset FROM test.kafka_{format_name}
WHERE length(_error) > 0;
"""
@ -4796,7 +4796,7 @@ def test_max_rows_per_message(kafka_cluster, create_query_generator):
DROP TABLE IF EXISTS test.kafka;
{create_query};
CREATE MATERIALIZED VIEW test.view Engine=Log AS
CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS
SELECT key, value FROM test.kafka;
"""
)
@ -4875,7 +4875,7 @@ def test_row_based_formats(kafka_cluster, create_query_generator):
{create_query};
CREATE MATERIALIZED VIEW test.view Engine=Log AS
CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS
SELECT key, value FROM test.{table_name};
INSERT INTO test.{table_name} SELECT number * 10 as key, number * 100 as value FROM numbers({num_rows});
@ -4982,7 +4982,7 @@ def test_block_based_formats_2(kafka_cluster, create_query_generator):
{create_query};
CREATE MATERIALIZED VIEW test.view Engine=Log AS
CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS
SELECT key, value FROM test.{table_name};
INSERT INTO test.{table_name} SELECT number * 10 as key, number * 100 as value FROM numbers({num_rows}) settings max_block_size=12, optimize_trivial_insert_select=0;
@ -5362,7 +5362,7 @@ def test_formats_errors(kafka_cluster):
input_format_with_names_use_header=0,
format_schema='key_value_message:Message';
CREATE MATERIALIZED VIEW test.view Engine=Log AS
CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS
SELECT key, value FROM test.{table_name};
"""
)

4
tests/integration/test_version_update/configs/log_conf.xml
View File

@ -1,4 +1,4 @@
<yandex>
<clickhouse>
<logger>
<level>trace</level>
<log>/var/log/clickhouse-server/log.log</log>
@ -8,4 +8,4 @@
<stderr>/var/log/clickhouse-server/stderr.log</stderr>
<stdout>/var/log/clickhouse-server/stdout.log</stdout>
</logger>
</yandex>
</clickhouse>

23
tests/performance/insert_select_squashing.xml Normal file
View File

@ -0,0 +1,23 @@
<test>
<settings>
<max_block_size>1000</max_block_size>
</settings>
<create_query>
CREATE TABLE squash_performance
(
s1 String,
s2 Nullable(String),
a1 Array(Array(String)),
a2 Array(Array(UInt32)),
m1 Map(String, Array(String)),
m2 Map(String, Array(UInt64)),
t Tuple(String, Array(String), Map(String, String))
)
ENGINE = Null;
</create_query>
<query>INSERT INTO squash_performance SELECT * FROM generateRandom(42) LIMIT 500000</query>
<drop_query>DROP TABLE IF EXISTS squash_performance</drop_query>
</test>

59
tests/performance/insert_select_squashing_dynamic.xml Normal file
View File

@ -0,0 +1,59 @@
<test>
<settings>
<max_block_size>1000</max_block_size>
<enable_named_columns_in_function_tuple>0</enable_named_columns_in_function_tuple>
<allow_experimental_dynamic_type>1</allow_experimental_dynamic_type>
</settings>
<create_query>
CREATE TABLE dynamic_squash_performance_1
(
d Dynamic
)
ENGINE = Null;
</create_query>
<create_query>
CREATE TABLE dynamic_squash_performance_2
(
d Dynamic(max_types=6)
)
ENGINE = Null;
</create_query>
<create_query>
CREATE TABLE src_dynamic_squash_performance_1
(
d Dynamic
)
ENGINE = Memory;
</create_query>
<create_query>
CREATE TABLE src_dynamic_squash_performance_2
(
d Dynamic(max_types=6)
)
ENGINE = Memory;
</create_query>
<fill_query>
<![CDATA[insert into src_dynamic_squash_performance_1 select multiIf(number % 15000 < 1000, multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b1v1 Array(UInt64))::Dynamic, number % 15 < 3,tuple(range(number % 100))::Tuple(b1v2 Array(UInt64))::Dynamic, number % 15 < 6,tuple(range(number % 100))::Tuple(b1v3 Array(UInt64))::Dynamic, number % 15 < 10,tuple(range(number % 100))::Tuple(b1v4 Array(UInt64))::Dynamic,tuple(range(number % 100))::Tuple(b1v5 Array(UInt64))::Dynamic), number % 15000 < 3000, multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b2v1 Array(UInt64))::Dynamic, number % 15 < 3,tuple(range(number % 100))::Tuple(b2v2 Array(UInt64))::Dynamic, number % 15 < 6,tuple(range(number % 100))::Tuple(b2v3 Array(UInt64))::Dynamic, number % 15 < 10,tuple(range(number % 100))::Tuple(b2v4 Array(UInt64))::Dynamic,tuple(range(number % 100))::Tuple(b2v5 Array(UInt64))::Dynamic), number % 15000 < 6000, multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b3v1 Array(UInt64))::Dynamic, number % 15 < 3,tuple(range(number % 100))::Tuple(b3v2 Array(UInt64))::Dynamic, number % 15 < 6,tuple(range(number % 100))::Tuple(b3v3 Array(UInt64))::Dynamic, number % 15 < 10,tuple(range(number % 100))::Tuple(b3v4 Array(UInt64))::Dynamic,tuple(range(number % 100))::Tuple(b3v5 Array(UInt64))::Dynamic), number % 15000 < 10000, multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b4v1 Array(UInt64))::Dynamic, number % 15 < 3,tuple(range(number % 100))::Tuple(b4v2 Array(UInt64))::Dynamic, number % 15 < 6,tuple(range(number % 100))::Tuple(b4v3 Array(UInt64))::Dynamic, number % 15 < 10,tuple(range(number % 100))::Tuple(b4v4 Array(UInt64))::Dynamic,tuple(range(number % 100))::Tuple(b4v5 Array(UInt64))::Dynamic), multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b5v1 Array(UInt64))::Dynamic, number % 15 < 3,tuple(range(number % 100))::Tuple(b5v2 Array(UInt64))::Dynamic, number % 15 < 6,tuple(range(number % 100))::Tuple(b5v3 Array(UInt64))::Dynamic, number % 15 < 10,tuple(range(number % 100))::Tuple(b5v4 Array(UInt64))::Dynamic,tuple(range(number % 100))::Tuple(b5v5 Array(UInt64))::Dynamic)) as res from numbers(1000000)]]>
</fill_query>
<fill_query>
<![CDATA[insert into src_dynamic_squash_performance_2 select multiIf(number % 15000 < 1000, multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b1v1 Array(UInt64))::Dynamic(max_types=6), number % 15 < 3,tuple(range(number % 100))::Tuple(b1v2 Array(UInt64))::Dynamic(max_types=6), number % 15 < 6,tuple(range(number % 100))::Tuple(b1v3 Array(UInt64))::Dynamic(max_types=6), number % 15 < 10,tuple(range(number % 100))::Tuple(b1v4 Array(UInt64))::Dynamic(max_types=6),tuple(range(number % 100))::Tuple(b1v5 Array(UInt64))::Dynamic(max_types=6)), number % 15000 < 3000, multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b2v1 Array(UInt64))::Dynamic(max_types=6), number % 15 < 3,tuple(range(number % 100))::Tuple(b2v2 Array(UInt64))::Dynamic(max_types=6), number % 15 < 6,tuple(range(number % 100))::Tuple(b2v3 Array(UInt64))::Dynamic(max_types=6), number % 15 < 10,tuple(range(number % 100))::Tuple(b2v4 Array(UInt64))::Dynamic(max_types=6),tuple(range(number % 100))::Tuple(b2v5 Array(UInt64))::Dynamic(max_types=6)), number % 15000 < 6000, multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b3v1 Array(UInt64))::Dynamic(max_types=6), number % 15 < 3,tuple(range(number % 100))::Tuple(b3v2 Array(UInt64))::Dynamic(max_types=6), number % 15 < 6,tuple(range(number % 100))::Tuple(b3v3 Array(UInt64))::Dynamic(max_types=6), number % 15 < 10,tuple(range(number % 100))::Tuple(b3v4 Array(UInt64))::Dynamic(max_types=6),tuple(range(number % 100))::Tuple(b3v5 Array(UInt64))::Dynamic(max_types=6)), number % 15000 < 10000, multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b4v1 Array(UInt64))::Dynamic(max_types=6), number % 15 < 3,tuple(range(number % 100))::Tuple(b4v2 Array(UInt64))::Dynamic(max_types=6), number % 15 < 6,tuple(range(number % 100))::Tuple(b4v3 Array(UInt64))::Dynamic(max_types=6), number % 15 < 10,tuple(range(number % 100))::Tuple(b4v4 Array(UInt64))::Dynamic(max_types=6),tuple(range(number % 100))::Tuple(b4v5 Array(UInt64))::Dynamic(max_types=6)), multiIf(number % 15 < 1,tuple(range(number % 100))::Tuple(b5v1 Array(UInt64))::Dynamic(max_types=6), number % 15 < 3,tuple(range(number % 100))::Tuple(b5v2 Array(UInt64))::Dynamic(max_types=6), number % 15 < 6,tuple(range(number % 100))::Tuple(b5v3 Array(UInt64))::Dynamic(max_types=6), number % 15 < 10,tuple(range(number % 100))::Tuple(b5v4 Array(UInt64))::Dynamic(max_types=6),tuple(range(number % 100))::Tuple(b5v5 Array(UInt64))::Dynamic(max_types=6))) as res from numbers(1000000)]]>
</fill_query>
<query>INSERT INTO dynamic_squash_performance_1 SELECT number::Dynamic FROM numbers(10000000)</query>
<query>INSERT INTO dynamic_squash_performance_1 SELECT range(number % 100)::Dynamic FROM numbers(2000000)</query>
<query>INSERT INTO dynamic_squash_performance_1 SELECT * FROM src_dynamic_squash_performance_1</query>
<query>INSERT INTO dynamic_squash_performance_2 SELECT * FROM src_dynamic_squash_performance_2</query>
<drop_query>DROP TABLE IF EXISTS dynamic_squash_performance_1</drop_query>
<drop_query>DROP TABLE IF EXISTS dynamic_squash_performance_2</drop_query>
<drop_query>DROP TABLE IF EXISTS src_dynamic_squash_performance_1</drop_query>
<drop_query>DROP TABLE IF EXISTS src_dynamic_squash_performance_2</drop_query>
</test>

34
tests/performance/insert_select_squashing_variant.xml Normal file
View File

@ -0,0 +1,34 @@
<test>
<settings>
<max_block_size>1000</max_block_size>
<enable_named_columns_in_function_tuple>0</enable_named_columns_in_function_tuple>
<allow_experimental_variant_type>1</allow_experimental_variant_type>
<allow_suspicious_variant_types>1</allow_suspicious_variant_types>
</settings>
<create_query>
CREATE TABLE variant_squash_performance
(
v Variant(Tuple(v1 Array(UInt64)), Tuple(v2 Array(UInt64)), Tuple(v3 Array(UInt64)), Tuple(v4 Array(UInt64)), Tuple(v5 Array(UInt64)))
)
ENGINE = Null;
</create_query>
<create_query>
CREATE TABLE src_variant_squash_performance
(
v Variant(Tuple(v1 Array(UInt64)), Tuple(v2 Array(UInt64)), Tuple(v3 Array(UInt64)), Tuple(v4 Array(UInt64)), Tuple(v5 Array(UInt64)))
)
ENGINE = Memory;
</create_query>
<fill_query>
<![CDATA[insert into src_variant_squash_performance with 'Variant(Tuple(v1 Array(UInt64)), Tuple(v2 Array(UInt64)), Tuple(v3 Array(UInt64)), Tuple(v4 Array(UInt64)), Tuple(v5 Array(UInt64)))' as variant_type select multiIf(number % 15 < 1, CAST(tuple(range(number % 100))::Tuple(v1 Array(UInt64)), variant_type), number % 15 < 3, CAST(tuple(range(number % 100))::Tuple(v2 Array(UInt64)), variant_type), number % 15 < 6, CAST(tuple(range(number % 100))::Tuple(v3 Array(UInt64)), variant_type), number % 15 < 10, CAST(tuple(range(number % 100))::Tuple(v4 Array(UInt64)), variant_type), CAST(tuple(range(number % 100))::Tuple(v5 Array(UInt64)), variant_type)) from numbers(1000000)]]>
</fill_query>
<query>INSERT INTO variant_squash_performance SELECT * FROM src_variant_squash_performance</query>
<drop_query>DROP TABLE IF EXISTS variant_squash_performance</drop_query>
<drop_query>DROP TABLE IF EXISTS src_variant_squash_performance</drop_query>
</test>

44
tests/queries/0_stateless/01172_transaction_counters.reference
View File

@ -16,25 +16,25 @@
7 all_3_3_0 (0,0,'00000000-0000-0000-0000-000000000000') 0
7 all_4_4_0 (0,0,'00000000-0000-0000-0000-000000000000') 0
8 1
1 1 AddPart 1 1 1 1 all_1_1_0
2 1 Begin 1 1 1 1
2 1 AddPart 1 1 1 1 all_2_2_0
2 1 Rollback 1 1 1 1
3 1 Begin 1 1 1 1
3 1 AddPart 1 1 1 1 all_3_3_0
3 1 Commit 1 1 1 0
1 1 LockPart 1 1 1 1 all_2_2_0
4 1 Begin 1 1 1 1
4 1 AddPart 1 1 1 1 all_4_4_0
4 1 Commit 1 1 1 0
5 1 Begin 1 1 1 1
5 1 AddPart 1 1 1 1 all_5_5_0
5 1 LockPart 1 1 1 1 all_1_1_0
5 1 LockPart 1 1 1 1 all_3_3_0
5 1 LockPart 1 1 1 1 all_4_4_0
5 1 LockPart 1 1 1 1 all_5_5_0
5 1 UnlockPart 1 1 1 1 all_1_1_0
5 1 UnlockPart 1 1 1 1 all_3_3_0
5 1 UnlockPart 1 1 1 1 all_4_4_0
5 1 UnlockPart 1 1 1 1 all_5_5_0
5 1 Rollback 1 1 1 1
1 AddPart 1 1 1 1 all_1_1_0
2 Begin 1 1 1 1
2 AddPart 1 1 1 1 all_2_2_0
2 Rollback 1 1 1 1
3 Begin 1 1 1 1
3 AddPart 1 1 1 1 all_3_3_0
3 Commit 1 1 1 0
1 LockPart 1 1 1 1 all_2_2_0
4 Begin 1 1 1 1
4 AddPart 1 1 1 1 all_4_4_0
4 Commit 1 1 1 0
5 Begin 1 1 1 1
5 AddPart 1 1 1 1 all_5_5_0
5 LockPart 1 1 1 1 all_1_1_0
5 LockPart 1 1 1 1 all_3_3_0
5 LockPart 1 1 1 1 all_4_4_0
5 LockPart 1 1 1 1 all_5_5_0
5 UnlockPart 1 1 1 1 all_1_1_0
5 UnlockPart 1 1 1 1 all_3_3_0
5 UnlockPart 1 1 1 1 all_4_4_0
5 UnlockPart 1 1 1 1 all_5_5_0
5 Rollback 1 1 1 1

1
tests/queries/0_stateless/01172_transaction_counters.sql
View File

@ -42,7 +42,6 @@ rollback;
system flush logs;
select indexOf((select arraySort(groupUniqArray(tid)) from system.transactions_info_log where database=currentDatabase() and table='txn_counters'), tid),
(toDecimal64(now64(6), 6) - toDecimal64(event_time, 6)) < 100,
type,
thread_id!=0,
length(query_id)=length(queryID()) or type='Commit' and query_id='', -- ignore fault injection after commit

10
tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh
View File

@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# $2 - query
function execute_query()
{
${CLICKHOUSE_CLIENT} --opentelemetry_start_trace_probability=1 --query_id $1 -nq "
${CLICKHOUSE_CLIENT} --opentelemetry_start_trace_probability=1 --query_id $1 -q "
${2}
"
}
@ -18,7 +18,7 @@ function execute_query()
# so we only to check the db.statement only
function check_query_span_query_only()
{
${CLICKHOUSE_CLIENT} -nq "
${CLICKHOUSE_CLIENT} -q "
SYSTEM FLUSH LOGS;
SELECT attribute['db.statement'] as query
FROM system.opentelemetry_span_log
@ -31,7 +31,7 @@ ${CLICKHOUSE_CLIENT} -nq "
function check_query_span()
{
${CLICKHOUSE_CLIENT} -nq "
${CLICKHOUSE_CLIENT} -q "
SYSTEM FLUSH LOGS;
SELECT attribute['db.statement'] as query,
attribute['clickhouse.read_rows'] as read_rows,
@ -47,7 +47,7 @@ ${CLICKHOUSE_CLIENT} -nq "
#
# Set up
#
${CLICKHOUSE_CLIENT} -nq "
${CLICKHOUSE_CLIENT} -q "
DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.opentelemetry_test;
CREATE TABLE ${CLICKHOUSE_DATABASE}.opentelemetry_test (id UInt64) Engine=MergeTree Order By id;
"
@ -79,4 +79,4 @@ check_query_span $query_id
#
${CLICKHOUSE_CLIENT} -q "
DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.opentelemetry_test;
"
"

2
tests/queries/0_stateless/02458_insert_select_progress_tcp.sh
View File

@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists insert_select_progress_tcp;
create table insert_select_progress_tcp(s UInt16) engine = MergeTree order by s;
"

8
tests/queries/0_stateless/02476_analyzer_identifier_hints.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS test_table;
CREATE TABLE test_table
(
@ -74,7 +74,7 @@ $CLICKHOUSE_CLIENT -q "SELECT 1 AS constant_value, arrayMap(lambda_argument -> l
$CLICKHOUSE_CLIENT -q "WITH 1 AS constant_value SELECT (SELECT constant_valu) SETTINGS enable_analyzer = 1;" 2>&1 \
| grep "Maybe you meant: \['constant_value'\]" &>/dev/null;
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS test_table_compound;
CREATE TABLE test_table_compound
(
@ -142,7 +142,7 @@ $CLICKHOUSE_CLIENT -q "SELECT cast(tuple(1), 'Tuple(value_1 String)') AS constan
$CLICKHOUSE_CLIENT -q "WITH cast(tuple(1), 'Tuple(value_1 String)') AS constant_value SELECT (SELECT constant_value.value_) SETTINGS enable_analyzer = 1;" 2>&1 \
| grep "Maybe you meant: \['constant_value.value_1'\]" &>/dev/null;
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS test_table_1;
CREATE TABLE test_table_1
(
@ -185,7 +185,7 @@ $CLICKHOUSE_CLIENT -q "SELECT ((1))::Tuple(a Tuple(b UInt32)) AS t, t.a.c SETTIN
$CLICKHOUSE_CLIENT -q "SELECT 1";
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE test_table;
DROP TABLE test_table_compound;
DROP TABLE test_table_1;

2
tests/queries/0_stateless/02480_tets_show_full.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
database=$($CLICKHOUSE_CLIENT -q 'SELECT currentDatabase()')
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE IF EXISTS test_02480_table;
DROP VIEW IF EXISTS test_02480_view;
CREATE TABLE test_02480_table (id Int64) ENGINE=MergeTree ORDER BY id;

2
tests/queries/0_stateless/02482_load_parts_refcounts.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query "
$CLICKHOUSE_CLIENT --query "
DROP TABLE IF EXISTS load_parts_refcounts SYNC;
CREATE TABLE load_parts_refcounts (id UInt32)

6
tests/queries/0_stateless/02496_remove_redundant_sorting.sh
View File

@ -26,15 +26,15 @@ FROM
ORDER BY number DESC
)
ORDER BY number ASC"
$CLICKHOUSE_CLIENT -nq "$DISABLE_OPTIMIZATION;EXPLAIN $query"
$CLICKHOUSE_CLIENT -q "$DISABLE_OPTIMIZATION;EXPLAIN $query"
function run_query {
echo "-- query"
echo "$1"
echo "-- explain"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;EXPLAIN $1"
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;EXPLAIN $1"
echo "-- execute"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;$1"
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;$1"
}
echo "-- Enabled query_plan_remove_redundant_sorting"

2
tests/queries/0_stateless/02497_storage_file_reader_selection.sh
View File

@ -13,6 +13,6 @@ $CLICKHOUSE_LOCAL --storage_file_read_method=mmap --print-profile-events -q "SEL
$CLICKHOUSE_LOCAL --storage_file_read_method=pread --print-profile-events -q "SELECT * FROM file($DATA_FILE) FORMAT Null" 2>&1 | grep -F -q "CreatedReadBufferMMap" && echo 'Fail' || echo 0
$CLICKHOUSE_LOCAL --storage_file_read_method=pread --print-profile-events -q "SELECT * FROM file($DATA_FILE) FORMAT Null" 2>&1 | grep -F -q "CreatedReadBufferOrdinary" && echo 1 || echo 'Fail'
$CLICKHOUSE_CLIENT --storage_file_read_method=mmap -nq "SELECT * FROM file('/dev/null', 'LineAsString') FORMAT Null -- { serverError BAD_ARGUMENTS }"
$CLICKHOUSE_CLIENT --storage_file_read_method=mmap -q "SELECT * FROM file('/dev/null', 'LineAsString') FORMAT Null -- { serverError BAD_ARGUMENTS }"
rm $DATA_FILE

6
tests/queries/0_stateless/02500_remove_redundant_distinct.sh
View File

@ -24,15 +24,15 @@ FROM
)
)"
$CLICKHOUSE_CLIENT -nq "$DISABLE_OPTIMIZATION;EXPLAIN $query"
$CLICKHOUSE_CLIENT -q "$DISABLE_OPTIMIZATION;EXPLAIN $query"
function run_query {
echo "-- query"
echo "$1"
echo "-- explain"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;EXPLAIN $1"
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;EXPLAIN $1"
echo "-- execute"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;$1"
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;$1"
}
echo "-- Enabled $OPTIMIZATION_SETTING"

4
tests/queries/0_stateless/02514_database_replicated_no_arguments_for_rmt.sh
View File

@ -14,8 +14,8 @@ ${CLICKHOUSE_CLIENT} -q "CREATE USER user_${CLICKHOUSE_DATABASE} settings databa
${CLICKHOUSE_CLIENT} -q "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}_db.* TO user_${CLICKHOUSE_DATABASE}"
${CLICKHOUSE_CLIENT} -q "GRANT TABLE ENGINE ON ReplicatedMergeTree TO user_${CLICKHOUSE_DATABASE}"
${CLICKHOUSE_CLIENT} -q "CREATE DATABASE ${CLICKHOUSE_DATABASE}_db engine = Replicated('/clickhouse/databases/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}_db', '{shard}', '{replica}')"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt_ok (x UInt32) engine = ReplicatedMergeTree order by x;"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt_fail (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/root/{shard}', '{replica}') order by x; -- { serverError 80 }"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt_ok (x UInt32) engine = ReplicatedMergeTree order by x;"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt_fail (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/root/{shard}', '{replica}') order by x; -- { serverError 80 }"
${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${CLICKHOUSE_DATABASE}_db"
${CLICKHOUSE_CLIENT} -q "DROP USER user_${CLICKHOUSE_DATABASE}"

2
tests/queries/0_stateless/02532_send_logs_level_test.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists data;
create table data (key Int) engine=MergeTree order by tuple() settings min_bytes_for_wide_part = '1G', compress_marks = 1;
insert into data values (1);

8
tests/queries/0_stateless/02555_davengers_rename_chain.sh
View File

@ -46,7 +46,7 @@ tables["wrong_metadata_compact"]="min_bytes_for_wide_part = 10000000"
for table in "${!tables[@]}"; do
settings="${tables[$table]}"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS $table;
CREATE TABLE $table(
@ -69,7 +69,7 @@ for table in "${!tables[@]}"; do
wait_column "$table" "\`a1\` UInt64" || exit 2
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
-- { echoOn }
SELECT 'ECHO_ALIGNMENT_FIX' FORMAT Null;
@ -82,7 +82,7 @@ for table in "${!tables[@]}"; do
wait_mutation_loaded "$table" "b1 TO a" || exit 2
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
-- { echoOn }
SELECT 'ECHO_ALIGNMENT_FIX' FORMAT Null;
@ -94,7 +94,7 @@ for table in "${!tables[@]}"; do
wait_for_all_mutations "$table"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
-- { echoOn }
SELECT 'ECHO_ALIGNMENT_FIX' FORMAT Null;

8
tests/queries/0_stateless/02572_query_views_log_background_thread.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "drop table if exists buffer_02572;
${CLICKHOUSE_CLIENT} --ignore-error --query "drop table if exists buffer_02572;
drop table if exists data_02572; drop table if exists copy_02572; drop table if exists mv_02572;"
${CLICKHOUSE_CLIENT} --query="create table copy_02572 (key Int) engine=Memory();"
@ -21,7 +21,7 @@ ${CLICKHOUSE_CLIENT} --query="insert into buffer_02572 values (1);"
if [ $(( $(date +%s) - start )) -gt 6 ]; then # clickhouse test cluster is overloaded, will skip
# ensure that the flush was not direct
${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "select * from data_02572; select * from copy_02572;"
${CLICKHOUSE_CLIENT} --ignore-error --query "select * from data_02572; select * from copy_02572;"
fi
# we cannot use OPTIMIZE, this will attach query context, so let's wait
@ -31,11 +31,11 @@ for _ in {1..100}; do
done
${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "select * from data_02572; select * from copy_02572;"
${CLICKHOUSE_CLIENT} --ignore-error --query "select * from data_02572; select * from copy_02572;"
${CLICKHOUSE_CLIENT} --query="system flush logs;"
${CLICKHOUSE_CLIENT} --query="select count() > 0, lower(status::String), errorCodeToName(exception_code)
from system.query_views_log where
view_name = concatWithSeparator('.', currentDatabase(), 'mv_02572') and
view_target = concatWithSeparator('.', currentDatabase(), 'copy_02572')
group by 2, 3;"
group by 2, 3;"

4
tests/queries/0_stateless/02703_max_local_read_bandwidth.sh
View File

@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists data;
create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9;
"
@ -26,7 +26,7 @@ read_methods=(
for read_method in "${read_methods[@]}"; do
query_id=$(random_str 10)
$CLICKHOUSE_CLIENT --query_id "$query_id" -q "select * from data format Null settings max_local_read_bandwidth='1M', local_filesystem_read_method='$read_method'"
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
SYSTEM FLUSH LOGS;
SELECT
'$read_method',

4
tests/queries/0_stateless/02703_max_local_write_bandwidth.sh
View File

@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists data;
create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9;
"
@ -13,7 +13,7 @@ $CLICKHOUSE_CLIENT -nm -q "
query_id=$(random_str 10)
# writes 1e6*8 bytes with 1M bandwith it should take (8-1)/1=7 seconds
$CLICKHOUSE_CLIENT --query_id "$query_id" -q "insert into data select * from numbers(1e6) settings max_local_write_bandwidth='1M'"
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
SYSTEM FLUSH LOGS;
SELECT
query_duration_ms >= 7e3,

4
tests/queries/0_stateless/02704_max_backup_bandwidth.sh
View File

@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists data;
create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9;
"
@ -15,7 +15,7 @@ $CLICKHOUSE_CLIENT -q "insert into data select * from numbers(1e6)"
query_id=$(random_str 10)
$CLICKHOUSE_CLIENT --query_id "$query_id" -q "backup table data to Disk('backups', '$CLICKHOUSE_DATABASE/data/backup1')" --max_backup_bandwidth=1M > /dev/null
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
SYSTEM FLUSH LOGS;
SELECT
query_duration_ms >= 7e3,

8
tests/queries/0_stateless/02724_limit_num_mutations.sh
View File

@ -23,7 +23,7 @@ function wait_for_alter()
done
}
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
DROP TABLE IF EXISTS t_limit_mutations SYNC;
CREATE TABLE t_limit_mutations (id UInt64, v UInt64)
@ -48,14 +48,14 @@ SELECT count() FROM system.mutations WHERE database = currentDatabase() AND tabl
SHOW CREATE TABLE t_limit_mutations;
"
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
ALTER TABLE t_limit_mutations UPDATE v = 6 WHERE 1 SETTINGS number_of_mutations_to_throw = 100;
ALTER TABLE t_limit_mutations MODIFY COLUMN v String SETTINGS number_of_mutations_to_throw = 100, alter_sync = 0;
"
wait_for_alter "String"
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
SELECT * FROM t_limit_mutations ORDER BY id;
SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_limit_mutations' AND NOT is_done;
SHOW CREATE TABLE t_limit_mutations;
@ -65,7 +65,7 @@ ${CLICKHOUSE_CLIENT} --query "SYSTEM START MERGES t_limit_mutations"
wait_for_mutation "t_limit_mutations" "0000000003"
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
SELECT * FROM t_limit_mutations ORDER BY id;
SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_limit_mutations' AND NOT is_done;
SHOW CREATE TABLE t_limit_mutations;

4
tests/queries/0_stateless/02725_async_insert_table_setting.sh
View File

@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
DROP TABLE IF EXISTS t_mt_async_insert;
DROP TABLE IF EXISTS t_mt_sync_insert;
@ -19,7 +19,7 @@ url="${CLICKHOUSE_URL}&async_insert=0&wait_for_async_insert=1"
${CLICKHOUSE_CURL} -sS "$url" -d "INSERT INTO t_mt_async_insert VALUES (1, 'aa'), (2, 'bb')"
${CLICKHOUSE_CURL} -sS "$url" -d "INSERT INTO t_mt_sync_insert VALUES (1, 'aa'), (2, 'bb')"
${CLICKHOUSE_CLIENT} -n --query "
${CLICKHOUSE_CLIENT} --query "
SELECT count() FROM t_mt_async_insert;
SELECT count() FROM t_mt_sync_insert;

4
tests/queries/0_stateless/02726_async_insert_flush_queue.sql
View File

@ -30,7 +30,9 @@ SELECT count() FROM t_async_inserts_flush;
SYSTEM FLUSH ASYNC INSERT QUEUE;
SELECT count() FROM system.asynchronous_inserts;
SELECT count() FROM system.asynchronous_inserts
WHERE database = currentDatabase() AND table = 't_async_inserts_flush';
SELECT count() FROM t_async_inserts_flush;
DROP TABLE t_async_inserts_flush;

2
tests/queries/0_stateless/02726_async_insert_flush_stress.sh
View File

@ -91,5 +91,5 @@ flush1 $TIMEOUT &
wait
${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH ASYNC INSERT QUEUE"
${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.asynchronous_inserts"
${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.asynchronous_inserts WHERE database = currentDatabase() AND table = 'async_inserts'"
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS async_inserts";

6
tests/queries/0_stateless/02841_local_assert.sh
View File

@ -7,12 +7,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CUR_DIR"/../shell_config.sh
echo "create table test (x UInt64) engine=Memory;
insert into test from infile 'data'; -- {clientError BAD_ARGUMENTS}" | $CLICKHOUSE_LOCAL -nm
insert into test from infile 'data'; -- {clientError BAD_ARGUMENTS}" | $CLICKHOUSE_LOCAL -m
echo "create table test (x UInt64) engine=Memory;
insert into test from infile 'data';" | $CLICKHOUSE_LOCAL -nm --ignore-error
insert into test from infile 'data';" | $CLICKHOUSE_LOCAL -m --ignore-error
echo "create table test (x UInt64) engine=Memory;
insert into test from infile 'data'; -- {clientError BAD_ARGUMENTS}
select 1" | $CLICKHOUSE_LOCAL -nm
select 1" | $CLICKHOUSE_LOCAL -m

14
tests/queries/0_stateless/02864_statistics_count_min_sketch.reference
View File

@ -1,14 +0,0 @@
CREATE TABLE default.tab\n(\n `a` String,\n `b` UInt64,\n `c` Int64,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
Test statistics count_min:
Prewhere info
Prewhere filter
Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
Test statistics multi-types:
Prewhere info
Prewhere filter
Prewhere filter column: and(equals(a, \'0\'), less(c, -90), greater(b, 900)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed)
Test LowCardinality and Nullable data type:
tab2

70
tests/queries/0_stateless/02864_statistics_count_min_sketch.sql
View File

@ -1,70 +0,0 @@
-- Tags: no-fasttest
DROP TABLE IF EXISTS tab SYNC;
SET allow_experimental_statistics = 1;
SET allow_statistics_optimize = 1;
SET allow_suspicious_low_cardinality_types=1;
SET mutations_sync = 2;
CREATE TABLE tab
(
a String,
b UInt64,
c Int64,
pk String,
) Engine = MergeTree() ORDER BY pk
SETTINGS min_bytes_for_wide_part = 0;
SHOW CREATE TABLE tab;
INSERT INTO tab select toString(number % 10000), number % 1000, -(number % 100), generateUUIDv4() FROM system.numbers LIMIT 10000;
SELECT 'Test statistics count_min:';
ALTER TABLE tab ADD STATISTICS a TYPE count_min;
ALTER TABLE tab ADD STATISTICS b TYPE count_min;
ALTER TABLE tab ADD STATISTICS c TYPE count_min;
ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx
WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
ALTER TABLE tab DROP STATISTICS a, b, c;
SELECT 'Test statistics multi-types:';
ALTER TABLE tab ADD STATISTICS a TYPE count_min;
ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq, tdigest;
ALTER TABLE tab ADD STATISTICS c TYPE count_min, uniq, tdigest;
ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
ALTER TABLE tab DROP STATISTICS a, b, c;
DROP TABLE IF EXISTS tab SYNC;
SELECT 'Test LowCardinality and Nullable data type:';
DROP TABLE IF EXISTS tab2 SYNC;
SET allow_suspicious_low_cardinality_types=1;
CREATE TABLE tab2
(
a LowCardinality(Int64) STATISTICS(count_min),
b Nullable(Int64) STATISTICS(count_min),
c LowCardinality(Nullable(Int64)) STATISTICS(count_min),
pk String,
) Engine = MergeTree() ORDER BY pk;
select name from system.tables where name = 'tab2' and database = currentDatabase();
DROP TABLE IF EXISTS tab2 SYNC;

37
tests/queries/0_stateless/02864_statistics_ddl.reference
View File

@ -1,31 +1,6 @@
CREATE TABLE default.tab\n(\n `a` Float64 STATISTICS(tdigest),\n `b` Int64 STATISTICS(tdigest),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
After insert
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), less(b, 10)) (removed)
10
0
After drop statistic
Prewhere info
Prewhere filter
Prewhere filter column: and(less(b, 10), less(a, 10)) (removed)
10
CREATE TABLE default.tab\n(\n `a` Float64,\n `b` Int64,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
After add statistic
CREATE TABLE default.tab\n(\n `a` Float64 STATISTICS(tdigest),\n `b` Int64 STATISTICS(tdigest),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
After materialize statistic
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), less(b, 10)) (removed)
20
After merge
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), less(b, 10)) (removed)
20
CREATE TABLE default.tab\n(\n `a` Float64 STATISTICS(tdigest),\n `c` Int64 STATISTICS(tdigest),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
After rename
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), less(c, 10)) (removed)
20
CREATE TABLE default.tab\n(\n `f64` Float64,\n `f64_tdigest` Float64 STATISTICS(tdigest),\n `f32` Float32,\n `s` String,\n `a` Array(Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
CREATE TABLE default.tab\n(\n `f64` Float64 STATISTICS(tdigest, uniq),\n `f64_tdigest` Float64 STATISTICS(tdigest),\n `f32` Float32 STATISTICS(tdigest, uniq),\n `s` String,\n `a` Array(Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
CREATE TABLE default.tab\n(\n `f64` Float64 STATISTICS(tdigest, uniq),\n `f64_tdigest` Float64 STATISTICS(tdigest),\n `f32` Float32 STATISTICS(tdigest, uniq),\n `s` String,\n `a` Array(Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
CREATE TABLE default.tab\n(\n `f64` Float64 STATISTICS(tdigest, uniq),\n `f64_tdigest` Float64 STATISTICS(tdigest),\n `f32` Float32 STATISTICS(tdigest, uniq),\n `s` String,\n `a` Array(Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
CREATE TABLE default.tab\n(\n `f64` Float64 STATISTICS(tdigest, uniq),\n `f64_tdigest` Float64 STATISTICS(tdigest),\n `f32` Float32 STATISTICS(tdigest, uniq),\n `s` String,\n `a` Array(Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
CREATE TABLE default.tab\n(\n `f64` Float64,\n `f64_tdigest` Float64 STATISTICS(tdigest),\n `f32` Float32,\n `s` String,\n `a` Array(Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192

234
tests/queries/0_stateless/02864_statistics_ddl.sql
View File

@ -1,59 +1,195 @@
-- Tests that various DDL statements create/drop/materialize statistics
-- Tags: no-fasttest
-- no-fasttest: 'count_min' sketches need a 3rd party library
-- Tests that DDL statements which create / drop / materialize statistics
SET mutations_sync = 1;
DROP TABLE IF EXISTS tab;
-- Error case: Can't create statistics when allow_experimental_statistics = 0
CREATE TABLE tab (col Float64 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY }
SET allow_experimental_statistics = 1;
SET allow_statistics_optimize = 1;
-- Error case: Unknown statistics types are rejected
CREATE TABLE tab (col Float64 STATISTICS(no_statistics_type)) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY }
-- Error case: The same statistics type can't exist more than once on a column
CREATE TABLE tab (col Float64 STATISTICS(tdigest, tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY }
SET allow_suspicious_low_cardinality_types = 1;
-- Statistics can only be created on columns of specific data types (depending on the statistics kind), (*)
-- tdigest requires data_type.isValueRepresentedByInteger
-- These types work:
CREATE TABLE tab (col UInt8 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col UInt256 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Float32 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Decimal32(3) STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Date STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Date32 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col DateTime STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col DateTime64 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Enum('hello', 'world') STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col IPv4 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Nullable(UInt8) STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col LowCardinality(UInt8) STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col LowCardinality(Nullable(UInt8)) STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
-- These types don't work:
CREATE TABLE tab (col String STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col FixedString(1) STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col Array(Float64) STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col Tuple(Float64, Float64) STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col UUID STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col IPv6 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
-- uniq requires data_type.isValueRepresentedByInteger
-- These types work:
CREATE TABLE tab (col UInt8 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col UInt256 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Float32 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Decimal32(3) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Date STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Date32 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col DateTime STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col DateTime64 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Enum('hello', 'world') STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col IPv4 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Nullable(UInt8) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col LowCardinality(UInt8) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col LowCardinality(Nullable(UInt8)) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
-- These types don't work:
CREATE TABLE tab (col String STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col FixedString(1) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col Array(Float64) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col Tuple(Float64, Float64) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col UUID STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col IPv6 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
-- count_min requires data_type.isValueRepresentedByInteger or data_type = (Fixed)String
-- These types work:
CREATE TABLE tab (col UInt8 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col UInt256 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Float32 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Decimal32(3) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Date STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Date32 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col DateTime STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col DateTime64 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Enum('hello', 'world') STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col IPv4 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col Nullable(UInt8) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col LowCardinality(UInt8) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col LowCardinality(Nullable(UInt8)) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col String STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
CREATE TABLE tab (col FixedString(1) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab;
-- These types don't work:
CREATE TABLE tab (col Array(Float64) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col Tuple(Float64, Float64) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col UUID STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab (col IPv6 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
-- CREATE TABLE was easy, ALTER is more fun
CREATE TABLE tab
(
a Float64 STATISTICS(tdigest),
b Int64 STATISTICS(tdigest),
pk String,
) Engine = MergeTree() ORDER BY pk
SETTINGS min_bytes_for_wide_part = 0;
f64 Float64,
f64_tdigest Float64 STATISTICS(tdigest),
f32 Float32,
s String,
a Array(Float64)
)
Engine = MergeTree()
ORDER BY tuple();
-- Error case: Unknown statistics types are rejected
-- (relevant for ADD and MODIFY)
ALTER TABLE tab ADD STATISTICS f64 TYPE no_statistics_type; -- { serverError INCORRECT_QUERY }
ALTER TABLE tab ADD STATISTICS IF NOT EXISTS f64 TYPE no_statistics_type; -- { serverError INCORRECT_QUERY }
ALTER TABLE tab MODIFY STATISTICS f64 TYPE no_statistics_type; -- { serverError INCORRECT_QUERY }
-- for some reason, ALTER TABLE tab MODIFY STATISTICS IF EXISTS is not supported
-- Error case: The same statistics type can't exist more than once on a column
-- (relevant for ADD and MODIFY)
-- Create the same statistics object twice
ALTER TABLE tab ADD STATISTICS f64 TYPE tdigest, tdigest; -- { serverError INCORRECT_QUERY }
ALTER TABLE tab ADD STATISTICS IF NOT EXISTS f64 TYPE tdigest, tdigest; -- { serverError INCORRECT_QUERY }
ALTER TABLE tab MODIFY STATISTICS f64 TYPE tdigest, tdigest; -- { serverError INCORRECT_QUERY }
-- Create an statistics which exists already
ALTER TABLE tab ADD STATISTICS f64_tdigest TYPE tdigest; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab ADD STATISTICS IF NOT EXISTS f64_tdigest TYPE tdigest; -- no-op
ALTER TABLE tab MODIFY STATISTICS f64_tdigest TYPE tdigest; -- no-op
-- Error case: Column does not exist
-- (relevant for ADD, MODIFY, DROP, CLEAR, and MATERIALIZE)
-- Note that the results are unfortunately quite inconsistent ...
ALTER TABLE tab ADD STATISTICS no_such_column TYPE tdigest; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab ADD STATISTICS IF NOT EXISTS no_such_column TYPE tdigest; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab MODIFY STATISTICS no_such_column TYPE tdigest; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab DROP STATISTICS no_such_column; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab DROP STATISTICS IF EXISTS no_such_column; -- no-op
ALTER TABLE tab CLEAR STATISTICS no_such_column; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab CLEAR STATISTICS IF EXISTS no_such_column; -- no-op
ALTER TABLE tab MATERIALIZE STATISTICS no_such_column; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab MATERIALIZE STATISTICS IF EXISTS no_such_column; -- { serverError ILLEGAL_STATISTICS }
-- Error case: Column exists but has no statistics
-- (relevant for MODIFY, DROP, CLEAR, and MATERIALIZE)
-- Note that the results are unfortunately quite inconsistent ...
ALTER TABLE tab MODIFY STATISTICS s TYPE tdigest; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab DROP STATISTICS s; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab DROP STATISTICS IF EXISTS s; -- no-op
ALTER TABLE tab CLEAR STATISTICS s; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab CLEAR STATISTICS IF EXISTS s; -- no-op
ALTER TABLE tab MATERIALIZE STATISTICS s; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab MATERIALIZE STATISTICS IF EXISTS s; -- { serverError ILLEGAL_STATISTICS }
-- We don't check systematically that that statistics can only be created via ALTER ADD STATISTICS on columns of specific data types (the
-- internal type validation code is tested already above, (*)). Only do a rudimentary check for each statistics type with a data type that
-- works and one that doesn't work.
-- tdigest
-- Works:
ALTER TABLE tab ADD STATISTICS f64 TYPE tdigest; ALTER TABLE tab DROP STATISTICS f64;
ALTER TABLE tab MODIFY STATISTICS f64 TYPE tdigest; ALTER TABLE tab DROP STATISTICS f64;
-- Doesn't work:
ALTER TABLE tab ADD STATISTICS a TYPE tdigest; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab MODIFY STATISTICS a TYPE tdigest; -- { serverError ILLEGAL_STATISTICS }
-- uniq
-- Works:
ALTER TABLE tab ADD STATISTICS f64 TYPE uniq; ALTER TABLE tab DROP STATISTICS f64;
ALTER TABLE tab MODIFY STATISTICS f64 TYPE count_min; ALTER TABLE tab DROP STATISTICS f64;
-- Doesn't work:
ALTER TABLE tab ADD STATISTICS a TYPE uniq; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab MODIFY STATISTICS a TYPE uniq; -- { serverError ILLEGAL_STATISTICS }
-- count_min
-- Works:
ALTER TABLE tab ADD STATISTICS f64 TYPE count_min; ALTER TABLE tab DROP STATISTICS f64;
ALTER TABLE tab MODIFY STATISTICS f64 TYPE count_min; ALTER TABLE tab DROP STATISTICS f64;
-- Doesn't work:
ALTER TABLE tab ADD STATISTICS a TYPE count_min; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab MODIFY STATISTICS a TYPE count_min; -- { serverError ILLEGAL_STATISTICS }
-- Any data type changes on columns with statistics are disallowed, for simplicity even if the new data type is compatible with all existing
-- statistics objects (e.g. tdigest can be created on Float64 and UInt64)
ALTER TABLE tab MODIFY COLUMN f64_tdigest UInt64; -- { serverError ALTER_OF_COLUMN_IS_FORBIDDEN }
-- Finally, do a full-circle test of a good case. Print table definition after each step.
-- Intentionally specifying _two_ columns and _two_ statistics types to have that also tested.
SHOW CREATE TABLE tab;
ALTER TABLE tab ADD STATISTICS f64, f32 TYPE tdigest, uniq;
SHOW CREATE TABLE tab;
ALTER TABLE tab MODIFY STATISTICS f64, f32 TYPE tdigest, uniq;
SHOW CREATE TABLE tab;
ALTER TABLE tab CLEAR STATISTICS f64, f32;
SHOW CREATE TABLE tab;
ALTER TABLE tab MATERIALIZE STATISTICS f64, f32;
SHOW CREATE TABLE tab;
ALTER TABLE tab DROP STATISTICS f64, f32;
SHOW CREATE TABLE tab;
INSERT INTO tab select number, -number, generateUUIDv4() FROM system.numbers LIMIT 10000;
SELECT 'After insert';
SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT count(*) FROM tab WHERE b < 10 and a < 10;
SELECT count(*) FROM tab WHERE b < NULL and a < '10';
ALTER TABLE tab DROP STATISTICS a, b;
SELECT 'After drop statistic';
SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT count(*) FROM tab WHERE b < 10 and a < 10;
SHOW CREATE TABLE tab;
ALTER TABLE tab ADD STATISTICS a, b TYPE tdigest;
SELECT 'After add statistic';
SHOW CREATE TABLE tab;
ALTER TABLE tab MATERIALIZE STATISTICS a, b;
INSERT INTO tab select number, -number, generateUUIDv4() FROM system.numbers LIMIT 10000;
SELECT 'After materialize statistic';
SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT count(*) FROM tab WHERE b < 10 and a < 10;
OPTIMIZE TABLE tab FINAL;
SELECT 'After merge';
SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT count(*) FROM tab WHERE b < 10 and a < 10;
ALTER TABLE tab RENAME COLUMN b TO c;
SHOW CREATE TABLE tab;
SELECT 'After rename';
SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 10 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT count(*) FROM tab WHERE c < 10 and a < 10;
DROP TABLE IF EXISTS tab;
DROP TABLE tab;

12
tests/queries/0_stateless/02864_statistics_delayed_materialization_in_merge.reference Normal file
View File

@ -0,0 +1,12 @@
After insert
Prewhere info
Prewhere filter
Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed)
After merge
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)
After truncate, insert, and materialize
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)

36
tests/queries/0_stateless/02864_statistics_delayed_materialization_in_merge.sql Normal file
View File

@ -0,0 +1,36 @@
-- Tests delayed materialization of statistics in merge instead of during insert (setting 'materialize_statistics_on_insert = 0').
-- (The concrete statistics type, column data type and predicate type don't matter)
-- Checks by the predicate evaluation order in EXPLAIN. This is quite fragile, a better approach would be helpful (maybe 'send_logs_level'?)
DROP TABLE IF EXISTS tab;
SET allow_experimental_statistics = 1;
SET allow_statistics_optimize = 1;
SET enable_analyzer = 1;
SET materialize_statistics_on_insert = 0;
CREATE TABLE tab
(
a Int64 STATISTICS(tdigest),
b Int16 STATISTICS(tdigest),
) ENGINE = MergeTree() ORDER BY tuple()
SETTINGS min_bytes_for_wide_part = 0, enable_vertical_merge_algorithm = 0; -- TODO: there is a bug in vertical merge with statistics.
INSERT INTO tab SELECT number, -number FROM system.numbers LIMIT 10000;
SELECT 'After insert';
SELECT replaceRegexpAll(explain, '__table1\.', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%'; -- checks b first, then a (statistics not used)
OPTIMIZE TABLE tab FINAL;
SELECT 'After merge';
SELECT replaceRegexpAll(explain, '__table1\.', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%'; -- checks a first, then b (statistics used)
TRUNCATE TABLE tab;
SET mutations_sync = 2;
INSERT INTO tab SELECT number, -number FROM system.numbers LIMIT 10000;
ALTER TABLE tab MATERIALIZE STATISTICS a, b;
SELECT 'After truncate, insert, and materialize';
SELECT replaceRegexpAll(explain, '__table1\.', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%'; -- checks a first, then b (statistics used)
DROP TABLE tab;

0
tests/queries/0_stateless/02864_statistics_exception.reference
View File

55
tests/queries/0_stateless/02864_statistics_exception.sql
View File

@ -1,55 +0,0 @@
-- Tests creating/dropping/materializing statistics produces the right exceptions.
DROP TABLE IF EXISTS tab;
-- Can't create statistics when allow_experimental_statistics = 0
CREATE TABLE tab
(
a Float64 STATISTICS(tdigest)
) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY }
SET allow_experimental_statistics = 1;
-- The same type of statistics can't exist more than once on a column
CREATE TABLE tab
(
a Float64 STATISTICS(tdigest, tdigest)
) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY }
-- Unknown statistics types are rejected
CREATE TABLE tab
(
a Float64 STATISTICS(no_statistics_type)
) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY }
-- tDigest statistics can only be created on numeric columns
CREATE TABLE tab
(
a String STATISTICS(tdigest),
) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS }
CREATE TABLE tab
(
a Float64,
b String
) Engine = MergeTree() ORDER BY tuple();
ALTER TABLE tab ADD STATISTICS a TYPE no_statistics_type; -- { serverError INCORRECT_QUERY }
ALTER TABLE tab ADD STATISTICS a TYPE tdigest;
ALTER TABLE tab ADD STATISTICS IF NOT EXISTS a TYPE tdigest;
ALTER TABLE tab ADD STATISTICS a TYPE tdigest; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab MODIFY STATISTICS a TYPE tdigest;
-- Statistics can be created only on integer columns
ALTER TABLE tab ADD STATISTICS b TYPE tdigest; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab DROP STATISTICS b; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab DROP STATISTICS a;
ALTER TABLE tab DROP STATISTICS IF EXISTS a;
ALTER TABLE tab CLEAR STATISTICS a; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab CLEAR STATISTICS IF EXISTS a;
ALTER TABLE tab MATERIALIZE STATISTICS b; -- { serverError ILLEGAL_STATISTICS }
ALTER TABLE tab ADD STATISTICS a TYPE tdigest;
ALTER TABLE tab MODIFY COLUMN a Float64 TTL toDateTime(b) + INTERVAL 1 MONTH;
ALTER TABLE tab MODIFY COLUMN a Int64; -- { serverError ALTER_OF_COLUMN_IS_FORBIDDEN }
DROP TABLE tab;

10
tests/queries/0_stateless/02864_statistics_materialize_in_merge.reference
View File

@ -1,10 +0,0 @@
10
10
10
statistics not used Condition less(b, 10_UInt8) moved to PREWHERE
statistics not used Condition less(a, 10_UInt8) moved to PREWHERE
statistics used after merge Condition less(a, 10_UInt8) moved to PREWHERE
statistics used after merge Condition less(b, 10_UInt8) moved to PREWHERE
statistics used after materialize Condition less(a, 10_UInt8) moved to PREWHERE
statistics used after materialize Condition less(b, 10_UInt8) moved to PREWHERE
2 0

52
tests/queries/0_stateless/02864_statistics_materialize_in_merge.sql
View File

@ -1,52 +0,0 @@
-- Tests delayed materialization of statistics in merge instead of during insert (setting 'materialize_statistics_on_insert = 0').
DROP TABLE IF EXISTS tab;
SET enable_analyzer = 1;
SET allow_experimental_statistics = 1;
SET allow_statistics_optimize = 1;
SET materialize_statistics_on_insert = 0;
CREATE TABLE tab
(
a Int64 STATISTICS(tdigest),
b Int16 STATISTICS(tdigest),
) ENGINE = MergeTree() ORDER BY tuple()
SETTINGS min_bytes_for_wide_part = 0, enable_vertical_merge_algorithm = 0; -- TODO: there is a bug in vertical merge with statistics.
INSERT INTO tab SELECT number, -number FROM system.numbers LIMIT 10000;
SELECT count(*) FROM tab WHERE b < 10 and a < 10 SETTINGS log_comment = 'statistics not used';
OPTIMIZE TABLE tab FINAL;
SELECT count(*) FROM tab WHERE b < 10 and a < 10 SETTINGS log_comment = 'statistics used after merge';
TRUNCATE TABLE tab;
SET mutations_sync = 2;
INSERT INTO tab SELECT number, -number FROM system.numbers LIMIT 10000;
ALTER TABLE tab MATERIALIZE STATISTICS a, b;
SELECT count(*) FROM tab WHERE b < 10 and a < 10 SETTINGS log_comment = 'statistics used after materialize';
DROP TABLE tab;
SYSTEM FLUSH LOGS;
SELECT log_comment, message FROM system.text_log JOIN
(
SELECT Settings['log_comment'] AS log_comment, query_id FROM system.query_log
WHERE current_database = currentDatabase()
AND query LIKE 'SELECT count(*) FROM tab%'
AND type = 'QueryFinish'
) AS query_log USING (query_id)
WHERE message LIKE '%moved to PREWHERE%'
ORDER BY event_time_microseconds;
SELECT count(), sum(ProfileEvents['MergeTreeDataWriterStatisticsCalculationMicroseconds'])
FROM system.query_log
WHERE current_database = currentDatabase()
AND query LIKE 'INSERT INTO tab SELECT%'
AND type = 'QueryFinish';

98
tests/queries/0_stateless/02864_statistics_predicates.reference Normal file
View File

@ -0,0 +1,98 @@
u64 and =
10
10
10
10
0
0
0
0
10
10
10
10
u64 and <
70
70
70
70
80
80
80
80
70
70
70
70
f64 and =
10
10
10
10
0
0
0
0
10
10
10
10
0
0
0
0
f64 and <
70
70
70
70
80
80
80
80
70
70
70
70
80
80
80
80
dt and =
0
0
0
0
10
10
10
10
dt and <
10000
10000
10000
10000
70
70
70
70
b and =
5000
5000
5000
5000
5000
5000
5000
5000
5000
5000
5000
5000
0
0
0
0
s and =
10
10

214
tests/queries/0_stateless/02864_statistics_predicates.sql Normal file
View File

@ -0,0 +1,214 @@
-- Tags: no-fasttest
-- no-fasttest: 'count_min' sketches need a 3rd party library
-- Tests the cross product of all predicates with all right-hand sides on all data types and all statistics types.
SET allow_experimental_statistics = 1;
SET allow_statistics_optimize = 1;
DROP TABLE IF EXISTS tab;
CREATE TABLE tab
(
u64 UInt64,
u64_tdigest UInt64 STATISTICS(tdigest),
u64_count_min UInt64 STATISTICS(count_min),
u64_uniq UInt64 STATISTICS(uniq),
f64 Float64,
f64_tdigest Float64 STATISTICS(tdigest),
f64_count_min Float64 STATISTICS(count_min),
f64_uniq Float64 STATISTICS(uniq),
dt DateTime,
dt_tdigest DateTime STATISTICS(tdigest),
dt_count_min DateTime STATISTICS(count_min),
dt_uniq DateTime STATISTICS(uniq),
b Bool,
b_tdigest Bool STATISTICS(tdigest),
b_count_min Bool STATISTICS(count_min),
b_uniq Bool STATISTICS(uniq),
s String,
-- s_tdigest String STATISTICS(tdigest), -- not supported by tdigest
s_count_min String STATISTICS(count_min)
-- s_uniq String STATISTICS(uniq), -- not supported by uniq
) Engine = MergeTree() ORDER BY tuple()
SETTINGS min_bytes_for_wide_part = 0;
INSERT INTO tab
-- SELECT number % 10000, number % 1000, -(number % 100) FROM system.numbers LIMIT 10000;
SELECT number % 1000,
number % 1000,
number % 1000,
number % 1000,
number % 1000,
number % 1000,
number % 1000,
number % 1000,
number % 1000,
number % 1000,
number % 1000,
number % 1000,
number % 2,
number % 2,
number % 2,
number % 2,
toString(number % 1000),
toString(number % 1000)
FROM system.numbers LIMIT 10000;
-- u64 ----------------------------------------------------
SELECT 'u64 and =';
SELECT count(*) FROM tab WHERE u64 = 7;
SELECT count(*) FROM tab WHERE u64_tdigest = 7;
SELECT count(*) FROM tab WHERE u64_count_min = 7;
SELECT count(*) FROM tab WHERE u64_uniq = 7;
SELECT count(*) FROM tab WHERE u64 = 7.7;
SELECT count(*) FROM tab WHERE u64_tdigest = 7.7;
SELECT count(*) FROM tab WHERE u64_count_min = 7.7;
SELECT count(*) FROM tab WHERE u64_uniq = 7.7;
SELECT count(*) FROM tab WHERE u64 = '7';
SELECT count(*) FROM tab WHERE u64_tdigest = '7';
SELECT count(*) FROM tab WHERE u64_count_min = '7';
SELECT count(*) FROM tab WHERE u64_uniq = '7';
SELECT count(*) FROM tab WHERE u64 = '7.7'; -- { serverError TYPE_MISMATCH }
SELECT count(*) FROM tab WHERE u64_tdigest = '7.7'; -- { serverError TYPE_MISMATCH }
SELECT count(*) FROM tab WHERE u64_count_min = '7.7'; -- { serverError TYPE_MISMATCH }
SELECT count(*) FROM tab WHERE u64_uniq = '7.7'; -- { serverError TYPE_MISMATCH }
SELECT 'u64 and <';
SELECT count(*) FROM tab WHERE u64 < 7;
SELECT count(*) FROM tab WHERE u64_tdigest < 7;
SELECT count(*) FROM tab WHERE u64_count_min < 7;
SELECT count(*) FROM tab WHERE u64_uniq < 7;
SELECT count(*) FROM tab WHERE u64 < 7.7;
SELECT count(*) FROM tab WHERE u64_tdigest < 7.7;
SELECT count(*) FROM tab WHERE u64_count_min < 7.7;
SELECT count(*) FROM tab WHERE u64_uniq < 7.7;
SELECT count(*) FROM tab WHERE u64 < '7';
SELECT count(*) FROM tab WHERE u64_tdigest < '7';
SELECT count(*) FROM tab WHERE u64_count_min < '7';
SELECT count(*) FROM tab WHERE u64_uniq < '7';
SELECT count(*) FROM tab WHERE u64 < '7.7'; -- { serverError TYPE_MISMATCH }
SELECT count(*) FROM tab WHERE u64_tdigest < '7.7'; -- { serverError TYPE_MISMATCH }
SELECT count(*) FROM tab WHERE u64_count_min < '7.7'; -- { serverError TYPE_MISMATCH }
SELECT count(*) FROM tab WHERE u64_uniq < '7.7'; -- { serverError TYPE_MISMATCH }
-- f64 ----------------------------------------------------
SELECT 'f64 and =';
SELECT count(*) FROM tab WHERE f64 = 7;
SELECT count(*) FROM tab WHERE f64_tdigest = 7;
SELECT count(*) FROM tab WHERE f64_count_min = 7;
SELECT count(*) FROM tab WHERE f64_uniq = 7;
SELECT count(*) FROM tab WHERE f64 = 7.7;
SELECT count(*) FROM tab WHERE f64_tdigest = 7.7;
SELECT count(*) FROM tab WHERE f64_count_min = 7.7;
SELECT count(*) FROM tab WHERE f64_uniq = 7.7;
SELECT count(*) FROM tab WHERE f64 = '7';
SELECT count(*) FROM tab WHERE f64_tdigest = '7';
SELECT count(*) FROM tab WHERE f64_count_min = '7';
SELECT count(*) FROM tab WHERE f64_uniq = '7';
SELECT count(*) FROM tab WHERE f64 = '7.7';
SELECT count(*) FROM tab WHERE f64_tdigest = '7.7';
SELECT count(*) FROM tab WHERE f64_count_min = '7.7';
SELECT count(*) FROM tab WHERE f64_uniq = '7.7';
SELECT 'f64 and <';
SELECT count(*) FROM tab WHERE f64 < 7;
SELECT count(*) FROM tab WHERE f64_tdigest < 7;
SELECT count(*) FROM tab WHERE f64_count_min < 7;
SELECT count(*) FROM tab WHERE f64_uniq < 7;
SELECT count(*) FROM tab WHERE f64 < 7.7;
SELECT count(*) FROM tab WHERE f64_tdigest < 7.7;
SELECT count(*) FROM tab WHERE f64_count_min < 7.7;
SELECT count(*) FROM tab WHERE f64_uniq < 7.7;
SELECT count(*) FROM tab WHERE f64 < '7';
SELECT count(*) FROM tab WHERE f64_tdigest < '7';
SELECT count(*) FROM tab WHERE f64_count_min < '7';
SELECT count(*) FROM tab WHERE f64_uniq < '7';
SELECT count(*) FROM tab WHERE f64 < '7.7';
SELECT count(*) FROM tab WHERE f64_tdigest < '7.7';
SELECT count(*) FROM tab WHERE f64_count_min < '7.7';
SELECT count(*) FROM tab WHERE f64_uniq < '7.7';
-- dt ----------------------------------------------------
SELECT 'dt and =';
SELECT count(*) FROM tab WHERE dt = '2024-08-08 11:12:13';
SELECT count(*) FROM tab WHERE dt_tdigest = '2024-08-08 11:12:13';
SELECT count(*) FROM tab WHERE dt_count_min = '2024-08-08 11:12:13';
SELECT count(*) FROM tab WHERE dt_uniq = '2024-08-08 11:12:13';
SELECT count(*) FROM tab WHERE dt = 7;
SELECT count(*) FROM tab WHERE dt_tdigest = 7;
SELECT count(*) FROM tab WHERE dt_count_min = 7;
SELECT count(*) FROM tab WHERE dt_uniq = 7;
SELECT 'dt and <';
SELECT count(*) FROM tab WHERE dt < '2024-08-08 11:12:13';
SELECT count(*) FROM tab WHERE dt_tdigest < '2024-08-08 11:12:13';
SELECT count(*) FROM tab WHERE dt_count_min < '2024-08-08 11:12:13';
SELECT count(*) FROM tab WHERE dt_uniq < '2024-08-08 11:12:13';
SELECT count(*) FROM tab WHERE dt < 7;
SELECT count(*) FROM tab WHERE dt_tdigest < 7;
SELECT count(*) FROM tab WHERE dt_count_min < 7;
SELECT count(*) FROM tab WHERE dt_uniq < 7;
-- b ----------------------------------------------------
SELECT 'b and =';
SELECT count(*) FROM tab WHERE b = true;
SELECT count(*) FROM tab WHERE b_tdigest = true;
SELECT count(*) FROM tab WHERE b_count_min = true;
SELECT count(*) FROM tab WHERE b_uniq = true;
SELECT count(*) FROM tab WHERE b = 'true';
SELECT count(*) FROM tab WHERE b_tdigest = 'true';
SELECT count(*) FROM tab WHERE b_count_min = 'true';
SELECT count(*) FROM tab WHERE b_uniq = 'true';
SELECT count(*) FROM tab WHERE b = 1;
SELECT count(*) FROM tab WHERE b_tdigest = 1;
SELECT count(*) FROM tab WHERE b_count_min = 1;
SELECT count(*) FROM tab WHERE b_uniq = 1;
SELECT count(*) FROM tab WHERE b = 1.1;
SELECT count(*) FROM tab WHERE b_tdigest = 1.1;
SELECT count(*) FROM tab WHERE b_count_min = 1.1;
SELECT count(*) FROM tab WHERE b_uniq = 1.1;
-- s ----------------------------------------------------
SELECT 's and =';
SELECT count(*) FROM tab WHERE s = 7; -- { serverError NO_COMMON_TYPE }
-- SELECT count(*) FROM tab WHERE s_tdigest = 7; -- not supported
SELECT count(*) FROM tab WHERE s_count_min = 7; -- { serverError NO_COMMON_TYPE }
-- SELECT count(*) FROM tab WHERE s_uniq = 7; -- not supported
SELECT count(*) FROM tab WHERE s = '7';
-- SELECT count(*) FROM tab WHERE s_tdigest = '7'; -- not supported
SELECT count(*) FROM tab WHERE s_count_min = '7';
-- SELECT count(*) FROM tab WHERE s_uniq = '7'; -- not supported
DROP TABLE tab;

35
tests/queries/0_stateless/02864_statistics_uniq.reference
View File

@ -1,35 +0,0 @@
CREATE TABLE default.t1\n(\n `a` Float64 STATISTICS(tdigest),\n `b` Int64 STATISTICS(tdigest),\n `c` Int64 STATISTICS(tdigest, uniq),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
After insert
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(equals(c, 11), less(a, 10), less(b, 10)) (removed)
After merge
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(equals(c, 11), less(a, 10), less(b, 10)) (removed)
After modify TDigest
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), equals(c, 11), less(b, 10)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(less(c, -1), less(a, 10), less(b, 10)) (removed)
After drop
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), equals(c, 11), less(b, 10)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10), less(c, -1), less(b, 10)) (removed)

73
tests/queries/0_stateless/02864_statistics_uniq.sql
View File

@ -1,73 +0,0 @@
DROP TABLE IF EXISTS t1;
SET allow_experimental_statistics = 1;
SET allow_statistics_optimize = 1;
SET mutations_sync = 1;
CREATE TABLE t1
(
a Float64 STATISTICS(tdigest),
b Int64 STATISTICS(tdigest),
c Int64 STATISTICS(tdigest, uniq),
pk String,
) Engine = MergeTree() ORDER BY pk
SETTINGS min_bytes_for_wide_part = 0;
SHOW CREATE TABLE t1;
INSERT INTO t1 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
INSERT INTO t1 select 0, 0, 11, generateUUIDv4();
SELECT 'After insert';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
OPTIMIZE TABLE t1 FINAL;
SELECT 'After merge';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT 'After modify TDigest';
ALTER TABLE t1 MODIFY STATISTICS c TYPE TDigest;
ALTER TABLE t1 MATERIALIZE STATISTICS c;
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
ALTER TABLE t1 DROP STATISTICS c;
SELECT 'After drop';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
DROP TABLE IF EXISTS t1;
DROP TABLE IF EXISTS t2;
SET allow_suspicious_low_cardinality_types=1;
CREATE TABLE t2
(
a Float64 STATISTICS(tdigest),
b Int64 STATISTICS(tdigest),
c LowCardinality(Int64) STATISTICS(tdigest, uniq),
pk String,
) Engine = MergeTree() ORDER BY pk
SETTINGS min_bytes_for_wide_part = 0;
INSERT INTO t2 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
DROP TABLE IF EXISTS t2;
DROP TABLE IF EXISTS t3;
CREATE TABLE t3
(
a Float64 STATISTICS(tdigest),
b Int64 STATISTICS(tdigest),
c Nullable(Int64) STATISTICS(tdigest, uniq),
pk String,
) Engine = MergeTree() ORDER BY pk
SETTINGS min_bytes_for_wide_part = 0;
INSERT INTO t3 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
DROP TABLE IF EXISTS t3;

20
tests/queries/0_stateless/02864_statistics_usage.reference Normal file
View File

@ -0,0 +1,20 @@
After insert
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)
After drop statistic
Prewhere info
Prewhere filter
Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed)
After add and materialize statistic
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)
After merge
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed)
After rename
Prewhere info
Prewhere filter
Prewhere filter column: and(less(a, 10_UInt8), less(c, 10_UInt8)) (removed)

42
tests/queries/0_stateless/02864_statistics_usage.sql Normal file
View File

@ -0,0 +1,42 @@
-- Test that the optimizer picks up column statistics
-- (The concrete statistics type, column data type and predicate type don't matter)
-- Checks by the predicate evaluation order in EXPLAIN. This is quite fragile, a better approach would be helpful (maybe 'send_logs_level'?)
SET allow_experimental_statistics = 1;
SET allow_statistics_optimize = 1;
SET mutations_sync = 1;
SET enable_analyzer = 1;
DROP TABLE IF EXISTS tab;
CREATE TABLE tab
(
a Float64 STATISTICS(tdigest),
b Int64 STATISTICS(tdigest)
) Engine = MergeTree() ORDER BY tuple()
SETTINGS min_bytes_for_wide_part = 0;
INSERT INTO tab select number, -number FROM system.numbers LIMIT 10000;
SELECT 'After insert';
SELECT replaceRegexpAll(explain, '__table1\.', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%'; -- checks a first, then b (statistics used)
ALTER TABLE tab DROP STATISTICS a, b;
SELECT 'After drop statistic';
SELECT replaceRegexpAll(explain, '__table1\.', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%'; -- checks b first, then a (statistics not used)
ALTER TABLE tab ADD STATISTICS a, b TYPE tdigest;
ALTER TABLE tab MATERIALIZE STATISTICS a, b;
INSERT INTO tab select number, -number FROM system.numbers LIMIT 10000;
SELECT 'After add and materialize statistic';
SELECT replaceRegexpAll(explain, '__table1\.', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%'; -- checks a first, then b (statistics used)
OPTIMIZE TABLE tab FINAL;
SELECT 'After merge';
SELECT replaceRegexpAll(explain, '__table1\.', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b < 10 and a < 10) WHERE explain LIKE '%Prewhere%'; -- checks a first, then b (statistics used)
ALTER TABLE tab RENAME COLUMN b TO c;
SELECT 'After rename';
SELECT replaceRegexpAll(explain, '__table1\.', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 10 and a < 10) WHERE explain LIKE '%Prewhere%'; -- checks a first, then c (statistics used)
DROP TABLE IF EXISTS tab;

6
tests/queries/0_stateless/02868_no_merge_across_partitions_final_with_lonely.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -nm -q """
${CLICKHOUSE_CLIENT} -m -q """
DROP TABLE IF EXISTS with_lonely;
CREATE TABLE with_lonely
@ -23,7 +23,7 @@ ORDER BY (id);
"""
create_optimize_partition() {
${CLICKHOUSE_CLIENT} -nm -q """
${CLICKHOUSE_CLIENT} -m -q """
INSERT INTO with_lonely SELECT number, '$1', number*10, 0 FROM numbers(10);
INSERT INTO with_lonely SELECT number+500000, '$1', number*10, 1 FROM numbers(10);
"""
@ -39,7 +39,7 @@ create_optimize_partition "2022-10-29"
create_optimize_partition "2022-10-30"
create_optimize_partition "2022-10-31"
${CLICKHOUSE_CLIENT} -nm -q """
${CLICKHOUSE_CLIENT} -m -q """
SYSTEM STOP MERGES with_lonely;
INSERT INTO with_lonely SELECT number, '2022-11-01', number*10, 0 FROM numbers(10);

2
tests/queries/0_stateless/02871_clickhouse_client_restart_pager.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# head by default print 10 rows, but it is not enough to query 11 rows, since
# we need to overflow the default pipe size, hence just 1 million of rows (it
# should be around 6 MiB in text representation, should be definitelly enough).
$CLICKHOUSE_CLIENT --ignore-error -nm --pager head -q "
$CLICKHOUSE_CLIENT --ignore-error -m --pager head -q "
select * from numbers(1e6); -- { clientError CANNOT_WRITE_TO_FILE_DESCRIPTOR }
select * from numbers(1e6); -- { clientError CANNOT_WRITE_TO_FILE_DESCRIPTOR }
"

4
tests/queries/0_stateless/02875_clickhouse_local_multiquery.sh
View File

@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT -q "select 1; select 2;"
$CLICKHOUSE_LOCAL -q "select 1; select 2;"
# -n is a no-op
$CLICKHOUSE_CLIENT -n -q "select 1; select 2;"
$CLICKHOUSE_LOCAL -n -q "select 1; select 2;"
$CLICKHOUSE_CLIENT -q "select 1; select 2;"
$CLICKHOUSE_LOCAL -q "select 1; select 2;"
exit 0

4
tests/queries/0_stateless/02875_merge_engine_set_index.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nq "
$CLICKHOUSE_CLIENT -q "
CREATE TABLE t1
(
a UInt32,
@ -57,7 +57,7 @@ ORDER BY
b DESC
FORMAT Null;"
$CLICKHOUSE_CLIENT -nq "
$CLICKHOUSE_CLIENT -q "
SYSTEM FLUSH LOGS;
SELECT ProfileEvents['SelectedMarks']

2
tests/queries/0_stateless/02878_use_structure_from_insertion_table_with_explicit_insert_columns.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_LOCAL -q "select 42 as x format Native" > $CLICKHOUSE_TEST_UNIQUE_NAME.native
$CLICKHOUSE_LOCAL -n -q "
$CLICKHOUSE_LOCAL -q "
create table test (x UInt64, y UInt64) engine=Memory;
insert into test (x) select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME.native');
insert into test (y) select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME.native');

2
tests/queries/0_stateless/02879_use_structure_from_insertion_table_with_defaults.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_LOCAL -q "select 1 as x format Native" > $CLICKHOUSE_TEST_UNIQUE_NAME.native
$CLICKHOUSE_LOCAL -n -q "
$CLICKHOUSE_LOCAL -q "
create table test (x UInt64, y UInt64 default 42) engine=Memory;
insert into test select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME.native');
select * from test;

2
tests/queries/0_stateless/02883_named_collections_override.sh
View File

@ -8,7 +8,7 @@ u1="${CLICKHOUSE_TEST_UNIQUE_NAME}_collection1"
u2="${CLICKHOUSE_TEST_UNIQUE_NAME}_collection2"
u3="${CLICKHOUSE_TEST_UNIQUE_NAME}_collection3"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
DROP NAMED COLLECTION IF EXISTS $u1;
DROP NAMED COLLECTION IF EXISTS $u2;

4
tests/queries/0_stateless/02884_async_insert_native_protocol_1.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
set -e
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS t_async_insert_native_1;
CREATE TABLE t_async_insert_native_1 (id UInt64, s String) ENGINE = MergeTree ORDER BY id;
"
@ -22,7 +22,7 @@ echo '{"id": 1, "s": "aaa"}' \
| $CLICKHOUSE_CLIENT $async_insert_options -q 'INSERT INTO t_async_insert_native_1 FORMAT JSONEachRow {"id": 2, "s": "bbb"}' 2>&1 \
| grep -o "NOT_IMPLEMENTED"
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
SELECT sum(length(entries.bytes)) FROM system.asynchronous_inserts
WHERE database = '$CLICKHOUSE_DATABASE' AND table = 't_async_insert_native_1';

4
tests/queries/0_stateless/02884_async_insert_native_protocol_2.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
set -e
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS t_async_insert_native_2;
CREATE TABLE t_async_insert_native_2 (id UInt64, s String) ENGINE = MergeTree ORDER BY id;
"
@ -18,7 +18,7 @@ echo "(3, 'ccc') (4, 'ddd') (5, 'eee')" | $CLICKHOUSE_CLIENT $async_insert_optio
wait
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
SELECT * FROM t_async_insert_native_2 ORDER BY id;
SYSTEM FLUSH LOGS;

4
tests/queries/0_stateless/02884_async_insert_native_protocol_3.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
set -e
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS t_async_insert_native_3;
CREATE TABLE t_async_insert_native_3 (id UInt64, s String) ENGINE = MergeTree ORDER BY id;
"
@ -21,7 +21,7 @@ $CLICKHOUSE_CLIENT $async_insert_options -q "INSERT INTO t_async_insert_native_3
wait
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
SELECT format, length(entries.bytes) FROM system.asynchronous_inserts
WHERE database = '$CLICKHOUSE_DATABASE' AND table = 't_async_insert_native_3'
ORDER BY format;

4
tests/queries/0_stateless/02884_async_insert_native_protocol_4.sh
View File

@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
DROP TABLE IF EXISTS t_async_insert_native_4;
CREATE TABLE t_async_insert_native_4 (id UInt64) ENGINE = MergeTree ORDER BY id;
"
@ -20,7 +20,7 @@ echo "(2) (3) (4) (5)" | $CLICKHOUSE_CLIENT_WITH_LOG $async_insert_options --asy
-q 'INSERT INTO t_async_insert_native_4 FORMAT Values' 2>&1 \
| grep -c "too much data"
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
SELECT * FROM t_async_insert_native_4 ORDER BY id;
SYSTEM FLUSH LOGS;

2
tests/queries/0_stateless/02885_ephemeral_columns_from_file.sh
View File

@ -9,7 +9,7 @@ $CLICKHOUSE_LOCAL -q "select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME.jsonl', a
$CLICKHOUSE_LOCAL -q "select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME.jsonl', auto, 'x UInt64 Alias y, y UInt64')" 2>&1 | grep -c "BAD_ARGUMENTS"
$CLICKHOUSE_LOCAL -q "select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME.jsonl', auto, 'x UInt64 Materialized 42, y UInt64')" 2>&1 | grep -c "BAD_ARGUMENTS"
$CLICKHOUSE_LOCAL -n -q "
$CLICKHOUSE_LOCAL -q "
create table test (x UInt64 Ephemeral, y UInt64 default x + 1) engine=Memory;
insert into test (x, y) select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME.jsonl');
select * from test;

2
tests/queries/0_stateless/02895_npy_output_format.sh
View File

@ -9,7 +9,7 @@ mkdir -p ${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}/
rm -rf ${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME:?}/*
chmod 777 ${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}/
${CLICKHOUSE_CLIENT} -n -q --ignore-error "
${CLICKHOUSE_CLIENT} -q --ignore-error "
DROP DATABASE IF EXISTS npy_output_02895;
CREATE DATABASE IF NOT EXISTS npy_output_02895;

2
tests/queries/0_stateless/02895_peak_memory_usage_http_headers_regression.sh
View File

@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE IF EXISTS data;
DROP TABLE IF EXISTS data2;
DROP VIEW IF EXISTS mv1;

4
tests/queries/0_stateless/02903_empty_order_by_throws_error.sh
View File

@ -5,13 +5,13 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CUR_DIR"/../shell_config.sh
# setting disabled and no order by or primary key; expect error
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS test_empty_order_by;
CREATE TABLE test_empty_order_by(a UInt8) ENGINE = MergeTree() SETTINGS index_granularity = 8192;
" 2>&1 \ | grep -F -q "You must provide an ORDER BY or PRIMARY KEY expression in the table definition." && echo 'OK' || echo 'FAIL'
# setting disabled and primary key in table definition
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS test_empty_order_by;
CREATE TABLE test_empty_order_by(a UInt8) ENGINE = MergeTree() PRIMARY KEY a SETTINGS index_granularity = 8192;
SHOW CREATE TABLE test_empty_order_by;

6
tests/queries/0_stateless/02903_rmt_retriable_merge_exception.sh
View File

@ -15,7 +15,7 @@ if [[ $($CLICKHOUSE_CLIENT -q "select count()>0 from system.clusters where clust
cluster=test_cluster_database_replicated
fi
$CLICKHOUSE_CLIENT -nm --distributed_ddl_output_mode=none -q "
$CLICKHOUSE_CLIENT -m --distributed_ddl_output_mode=none -q "
drop table if exists rmt1;
drop table if exists rmt2;
@ -46,7 +46,7 @@ part_name='%'
# wait while there be at least one 'No active replica has part all_0_1_1 or covering part' in logs
for _ in {0..50}; do
no_active_repilica_messages=$($CLICKHOUSE_CLIENT -nm -q "
no_active_repilica_messages=$($CLICKHOUSE_CLIENT -m -q "
system flush logs;
select count()
@ -65,7 +65,7 @@ for _ in {0..50}; do
sleep 1
done
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
system start pulling replication log rmt2;
system flush logs;

8
tests/queries/0_stateless/02904_empty_order_by_with_setting_enabled.sh
View File

@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CUR_DIR"/../shell_config.sh
# setting enabled and no order by or primary key
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET create_table_empty_primary_key_by_default = true;
DROP TABLE IF EXISTS test_empty_order_by;
CREATE TABLE test_empty_order_by(a UInt8) ENGINE = MergeTree() SETTINGS index_granularity = 8192;
@ -13,7 +13,7 @@ ${CLICKHOUSE_CLIENT} -n --query="
" 2>&1 \ | grep -F -q "ORDER BY tuple()" && echo 'OK' || echo 'FAIL'
# setting enabled and per-column primary key
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET create_table_empty_primary_key_by_default = true;
DROP TABLE IF EXISTS test_empty_order_by;
CREATE TABLE test_empty_order_by(a UInt8 PRIMARY KEY, b String PRIMARY KEY) ENGINE = MergeTree() SETTINGS index_granularity = 8192;
@ -21,7 +21,7 @@ ${CLICKHOUSE_CLIENT} -n --query="
" 2>&1 \ | grep -F -q "ORDER BY (a, b)" && echo 'OK' || echo 'FAIL'
# setting enabled and primary key in table definition (not per-column or order by)
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET create_table_empty_primary_key_by_default = true;
DROP TABLE IF EXISTS test_empty_order_by;
CREATE TABLE test_empty_order_by(a UInt8, b String) ENGINE = MergeTree() PRIMARY KEY (a) SETTINGS index_granularity = 8192;
@ -29,7 +29,7 @@ ${CLICKHOUSE_CLIENT} -n --query="
" 2>&1 \ | grep -F -q "ORDER BY a" && echo 'OK' || echo 'FAIL'
# setting enabled and order by in table definition (no primary key)
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET create_table_empty_primary_key_by_default = true;
DROP TABLE IF EXISTS test_empty_order_by;
CREATE TABLE test_empty_order_by(a UInt8, b String) ENGINE = MergeTree() ORDER BY (a, b) SETTINGS index_granularity = 8192;

8
tests/queries/0_stateless/02907_backup_mv_with_no_inner_table.sh
View File

@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table if exists src;
create table src (a Int32) engine = MergeTree() order by tuple();
@ -15,14 +15,14 @@ create materialized view mv (a Int32) engine = MergeTree() order by tuple() as s
uuid=$(${CLICKHOUSE_CLIENT} --query "select uuid from system.tables where table='mv' and database == currentDatabase()")
inner_table=".inner_id.${uuid}"
${CLICKHOUSE_CLIENT} -nm --query "drop table \`$inner_table\` sync"
${CLICKHOUSE_CLIENT} -m --query "drop table \`$inner_table\` sync"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
set send_logs_level = 'error';
backup table ${CLICKHOUSE_DATABASE}.\`mv\` to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}');
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table mv;
restore table ${CLICKHOUSE_DATABASE}.\`mv\` from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}');
" | grep -o "RESTORED"

12
tests/queries/0_stateless/02907_backup_mv_with_no_source_table.sh
View File

@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table if exists src;
create table src (a Int32) engine = MergeTree() order by tuple();
@ -15,18 +15,18 @@ drop table if exists mv;
create materialized view mv to dst (a Int32) as select * from src;
"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table src;
backup database ${CLICKHOUSE_DATABASE} on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}');
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table mv;
set allow_deprecated_database_ordinary=1;
restore table ${CLICKHOUSE_DATABASE}.mv on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}');
" | grep -o "RESTORED"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table if exists src;
create table src (a Int32) engine = MergeTree() order by tuple();
@ -37,13 +37,13 @@ drop table if exists mv;
create materialized view mv to dst (a Int32) as select * from src;
"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table src;
drop table dst;
backup database ${CLICKHOUSE_DATABASE} on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}2');
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table mv;
set allow_deprecated_database_ordinary=1;
restore table ${CLICKHOUSE_DATABASE}.mv on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}2');

4
tests/queries/0_stateless/02907_backup_restore_default_nullable.sh
View File

@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table if exists test;
set data_type_default_nullable = 0;
create table test (test String) ENGINE = MergeTree() ORDER BY tuple();
@ -13,7 +13,7 @@ backup table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost to Disk
${CLICKHOUSE_CLIENT} --query "show create table test"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table test sync;
set data_type_default_nullable = 1;
restore table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}');

8
tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh
View File

@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table if exists test;
set flatten_nested = 0;
create table test (test Array(Tuple(foo String, bar Float64))) ENGINE = MergeTree() ORDER BY tuple();
@ -13,7 +13,7 @@ backup table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost to Disk
${CLICKHOUSE_CLIENT} --query "show create table test"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table if exists test2;
set flatten_nested = 0;
create table test2 (test Nested(foo String, bar Float64)) ENGINE = MergeTree() ORDER BY tuple();
@ -22,7 +22,7 @@ backup table ${CLICKHOUSE_DATABASE}.test2 on cluster test_shard_localhost to Dis
${CLICKHOUSE_CLIENT} --query "show create table test2"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table test sync;
set flatten_nested = 1;
restore table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}');
@ -30,7 +30,7 @@ restore table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost from D
${CLICKHOUSE_CLIENT} --query "show create table test"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table test2 sync;
set flatten_nested = 1;
restore table ${CLICKHOUSE_DATABASE}.test2 on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}2');

2
tests/queries/0_stateless/02907_clickhouse_dictionary_bug.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
DROP DICTIONARY IF EXISTS 02907_dictionary;
DROP TABLE IF EXISTS 02907_table;

6
tests/queries/0_stateless/02907_system_backups_profile_events.sh
View File

@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
drop table if exists test;
create table test (a Int32) engine = MergeTree() order by tuple();
"
@ -12,10 +12,10 @@ create table test (a Int32) engine = MergeTree() order by tuple();
backup_id=${CLICKHOUSE_TEST_UNIQUE_NAME}
backup_name="Disk('backups', '$backup_id')";
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
backup table ${CLICKHOUSE_DATABASE}.test to $backup_name;
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
select ProfileEvents['BackupEntriesCollectorMicroseconds'] > 10 from system.backups where name='Disk(\'backups\', \'$backup_id\')'
"

4
tests/queries/0_stateless/02908_Npy_files_caching.sh
View File

@ -7,13 +7,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_LOCAL -q "select count() from file('$CURDIR/data_npy/one_dim.npy') settings optimize_count_from_files=0"
$CLICKHOUSE_LOCAL -q "select count() from file('$CURDIR/data_npy/one_dim.npy') settings optimize_count_from_files=1"
$CLICKHOUSE_LOCAL -q "select count() from file('$CURDIR/data_npy/one_dim.npy', auto, 'array Int64') settings optimize_count_from_files=1"
$CLICKHOUSE_LOCAL -nm -q "
$CLICKHOUSE_LOCAL -m -q "
desc file('$CURDIR/data_npy/one_dim.npy');
select number_of_rows from system.schema_inference_cache where format='Npy';
"
$CLICKHOUSE_LOCAL -q "select count() from file('$CURDIR/data_npy/npy_big.npy') settings optimize_count_from_files=0"
$CLICKHOUSE_LOCAL -q "select count() from file('$CURDIR/data_npy/npy_big.npy') settings optimize_count_from_files=1"
$CLICKHOUSE_LOCAL -nm -q "
$CLICKHOUSE_LOCAL -m -q "
desc file('$CURDIR/data_npy/npy_big.npy');
select number_of_rows from system.schema_inference_cache where format='Npy';
"

6
tests/queries/0_stateless/02908_table_ttl_dependency.sh
View File

@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE IF EXISTS 02908_dependent;
DROP TABLE IF EXISTS 02908_main;
@ -14,11 +14,11 @@ $CLICKHOUSE_CLIENT -nm -q "
CREATE TABLE 02908_dependent (a UInt32, ts DateTime) ENGINE = MergeTree ORDER BY a TTL ts + 1 WHERE a IN (SELECT a FROM ${CLICKHOUSE_DATABASE}.02908_main);
"
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE 02908_main;
" 2>&1 | grep -F -q "HAVE_DEPENDENT_OBJECTS"
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE 02908_dependent;
DROP TABLE 02908_main;
"

2
tests/queries/0_stateless/02909_settings_in_json_schema_cache.sh
View File

@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CUR_DIR"/../shell_config.sh
echo '{"x" : 42}' > $CLICKHOUSE_TEST_UNIQUE_NAME.json
$CLICKHOUSE_LOCAL -nm -q "
$CLICKHOUSE_LOCAL -m -q "
DESC file('$CLICKHOUSE_TEST_UNIQUE_NAME.json') SETTINGS schema_inference_make_columns_nullable=1;
DESC file('$CLICKHOUSE_TEST_UNIQUE_NAME.json') SETTINGS schema_inference_make_columns_nullable=0;
SELECT count() from system.schema_inference_cache where format = 'JSON' and additional_format_info like '%schema_inference_make_columns_nullable%';"

2
tests/queries/0_stateless/02915_input_table_function_in_subquery.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
CREATE TABLE IF NOT EXISTS ts_data_double_raw
(
device_id UInt32 NOT NULL CODEC(ZSTD),

26
tests/queries/0_stateless/02915_lazy_loading_of_base_backups.sh
View File

@ -13,40 +13,40 @@ b_backup="Disk('backups', '$b_backup_id')"
c_backup_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_c
c_backup="Disk('backups', '$c_backup_id')"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
DROP TABLE IF EXISTS tbl1;
DROP TABLE IF EXISTS tbl2;
DROP TABLE IF EXISTS tbl3;
"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
CREATE TABLE tbl1 (a Int32) ENGINE = MergeTree() ORDER BY tuple();
"
# The following BACKUP command must write backup 'a'.
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
BACKUP DATABASE ${CLICKHOUSE_DATABASE} TO $a_backup SETTINGS id='$a_backup_id';
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
CREATE TABLE tbl2 (a Int32) ENGINE = MergeTree() ORDER BY tuple();
"
# The following BACKUP command must read backup 'a' and write backup 'b'.
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
BACKUP DATABASE ${CLICKHOUSE_DATABASE} TO $b_backup SETTINGS id='$b_backup_id', base_backup=$a_backup;
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
CREATE TABLE tbl3 (a Int32) ENGINE = MergeTree() ORDER BY tuple();
"
# The following BACKUP command must read only backup 'b' (and not 'a') and write backup 'c'.
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
BACKUP DATABASE ${CLICKHOUSE_DATABASE} TO $c_backup SETTINGS id='$c_backup_id', base_backup=$b_backup;
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
DROP TABLE tbl1;
DROP TABLE tbl2;
DROP TABLE tbl3;
@ -57,28 +57,28 @@ r2_restore_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_r2
r3_restore_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_r3
# The following RESTORE command must read all 3 backups 'a', 'b', c' because the table 'tbl1' was in the first backup.
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
RESTORE TABLE ${CLICKHOUSE_DATABASE}.tbl1 FROM $c_backup SETTINGS id='$r1_restore_id';
" | grep -o "RESTORED"
# The following RESTORE command must read only 2 backups 'b', c' (and not 'a') because the table 'tbl2' was in the second backup.
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
RESTORE TABLE ${CLICKHOUSE_DATABASE}.tbl2 FROM $c_backup SETTINGS id='$r2_restore_id';
" | grep -o "RESTORED"
# The following RESTORE command must read only 1 backup 'c' (and not 'a' or 'b') because the table 'tbl3' was in the third backup.
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
RESTORE TABLE ${CLICKHOUSE_DATABASE}.tbl3 FROM $c_backup SETTINGS id='$r3_restore_id';
" | grep -o "RESTORED"
all_ids="['$a_backup_id', '$b_backup_id', '$c_backup_id', '$r1_restore_id', '$r2_restore_id', '$r3_restore_id']"
id_prefix_len=`expr "${CLICKHOUSE_TEST_UNIQUE_NAME}_" : '.*'`
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
SELECT substr(id, 1 + $id_prefix_len) as short_id, ProfileEvents['BackupsOpenedForRead'], ProfileEvents['BackupsOpenedForWrite'] FROM system.backups WHERE id IN ${all_ids} ORDER BY short_id
"
${CLICKHOUSE_CLIENT} -nm --query "
${CLICKHOUSE_CLIENT} -m --query "
DROP TABLE tbl1;
DROP TABLE tbl2;
DROP TABLE tbl3;

Some files were not shown because too many files have changed in this diff Show More