mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Merge branch 'master' into add-setting-allow-head-object-request
This commit is contained in:
alesapin
GitHub
commit
631c8fb155
2013
CHANGELOG.md
2013
CHANGELOG.md
File diff suppressed because it is too large
Load Diff
@ -16,6 +16,5 @@ ClickHouse® is an open-source column-oriented database management system that a
|
||||
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
|
||||
|
||||
## Upcoming events
|
||||
* **Recording available**: [**v22.12 Release Webinar**](https://www.youtube.com/watch?v=sREupr6uc2k) 22.12 is the ClickHouse Christmas release. There are plenty of gifts (a new JOIN algorithm among them) and we adopted something from MongoDB. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
|
||||
* [**ClickHouse Meetup at the CHEQ office in Tel Aviv**](https://www.meetup.com/clickhouse-tel-aviv-user-group/events/289599423/) - Jan 16 - We are very excited to be holding our next in-person ClickHouse meetup at the CHEQ office in Tel Aviv! Hear from CHEQ, ServiceNow and Contentsquare, as well as a deep dive presentation from ClickHouse CTO Alexey Milovidov. Join us for a fun evening of talks, food and discussion!
|
||||
* [**ClickHouse Meetup at Microsoft Office in Seattle**](https://www.meetup.com/clickhouse-seattle-user-group/events/290310025/) - Jan 18 - Keep an eye on this space as we will be announcing speakers soon!
|
||||
* **Recording available**: [**v23.1 Release Webinar**](https://www.youtube.com/watch?v=zYSZXBnTMSE) 23.1 is the ClickHouse New Year release. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release. Inverted indices, query cache, and so -- very -- much more.
|
||||
* **Recording available**: [**ClickHouse Meetup at the CHEQ office in Tel Aviv**](https://www.meetup.com/clickhouse-tel-aviv-user-group/events/289599423/) - We are very excited to be holding our next in-person ClickHouse meetup at the CHEQ office in Tel Aviv! Hear from CHEQ, ServiceNow and Contentsquare, as well as a deep dive presentation from ClickHouse CTO Alexey Milovidov. Join us for a fun evening of talks, food and discussion!
|
||||
|
||||
@ -138,6 +138,7 @@ function clone_submodules
|
||||
contrib/c-ares
|
||||
contrib/morton-nd
|
||||
contrib/xxHash
|
||||
contrib/simdjson
|
||||
)
|
||||
|
||||
git submodule sync
|
||||
@ -158,6 +159,7 @@ function run_cmake
|
||||
"-DENABLE_THINLTO=0"
|
||||
"-DUSE_UNWIND=1"
|
||||
"-DENABLE_NURAFT=1"
|
||||
"-DENABLE_SIMDJSON=1"
|
||||
"-DENABLE_JEMALLOC=1"
|
||||
)
|
||||
|
||||
@ -234,6 +236,7 @@ function run_tests
|
||||
--check-zookeeper-session
|
||||
--order random
|
||||
--print-time
|
||||
--report-logs-stats
|
||||
--jobs "${NPROC}"
|
||||
)
|
||||
time clickhouse-test "${test_opts[@]}" -- "$FASTTEST_FOCUS" 2>&1 \
|
||||
|
||||
@ -528,6 +528,7 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
-e "MutateFromLogEntryTask" \
|
||||
-e "No connection to ZooKeeper, cannot get shared table ID" \
|
||||
-e "Session expired" \
|
||||
-e "TOO_MANY_PARTS" \
|
||||
/var/log/clickhouse-server/clickhouse-server.backward.dirty.log | rg -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
|
||||
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
@ -923,15 +923,25 @@ Configuration markup:
|
||||
<single_read_retries>4</single_read_retries>
|
||||
<min_bytes_for_seek>1000</min_bytes_for_seek>
|
||||
<metadata_path>/var/lib/clickhouse/disks/s3/</metadata_path>
|
||||
<cache_enabled>true</cache_enabled>
|
||||
<cache_path>/var/lib/clickhouse/disks/s3/cache/</cache_path>
|
||||
<skip_access_check>false</skip_access_check>
|
||||
</s3>
|
||||
<s3_cache>
|
||||
<type>cache</type>
|
||||
<disk>s3</disk>
|
||||
<path>/var/lib/clickhouse/disks/s3_cache/</path>
|
||||
<max_size>10Gi</max_size>
|
||||
</s3_cache>
|
||||
</disks>
|
||||
...
|
||||
</storage_configuration>
|
||||
```
|
||||
|
||||
:::note cache configuration
|
||||
ClickHouse versions 22.3 through 22.7 use a different cache configuration, see [using local cache](/docs/en/operations/storing-data.md/#using-local-cache) if you are using one of those versions.
|
||||
:::
|
||||
|
||||
### Configuring the S3 disk
|
||||
|
||||
Required parameters:
|
||||
|
||||
- `endpoint` — S3 endpoint URL in `path` or `virtual hosted` [styles](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). Endpoint URL should contain a bucket and root path to store data.
|
||||
@ -951,8 +961,6 @@ Optional parameters:
|
||||
- `single_read_retries` — Number of retry attempts in case of connection drop during read. Default value is `4`.
|
||||
- `min_bytes_for_seek` — Minimal number of bytes to use seek operation instead of sequential read. Default value is `1 Mb`.
|
||||
- `metadata_path` — Path on local FS to store metadata files for S3. Default value is `/var/lib/clickhouse/disks/<disk_name>/`.
|
||||
- `cache_enabled` — Allows to cache mark and index files on local FS. Default value is `true`.
|
||||
- `cache_path` — Path on local FS where to store cached mark and index files. Default value is `/var/lib/clickhouse/disks/<disk_name>/cache/`.
|
||||
- `skip_access_check` — If true, disk access checks will not be performed on disk start-up. Default value is `false`.
|
||||
- `server_side_encryption_customer_key_base64` — If specified, required headers for accessing S3 objects with SSE-C encryption will be set.
|
||||
- `s3_max_put_rps` — Maximum PUT requests per second rate before throttling. Default value is `0` (unlimited).
|
||||
@ -960,6 +968,30 @@ Optional parameters:
|
||||
- `s3_max_get_rps` — Maximum GET requests per second rate before throttling. Default value is `0` (unlimited).
|
||||
- `s3_max_get_burst` — Max number of requests that can be issued simultaneously before hitting request per second limit. By default (`0` value) equals to `s3_max_get_rps`.
|
||||
|
||||
### Configuring the cache
|
||||
|
||||
This is the cache configuration from above:
|
||||
```xml
|
||||
<s3_cache>
|
||||
<type>cache</type>
|
||||
<disk>s3</disk>
|
||||
<path>/var/lib/clickhouse/disks/s3_cache/</path>
|
||||
<max_size>10Gi</max_size>
|
||||
</s3_cache>
|
||||
```
|
||||
|
||||
These parameters define the cache layer:
|
||||
- `type` — If a disk is of type `cache` it caches mark and index files in memory.
|
||||
- `disk` — The name of the disk that will be cached.
|
||||
|
||||
Cache parameters:
|
||||
- `path` — The path where metadata for the cache is stored.
|
||||
- `max_size` — The size (amount of memory) that the cache can grow to.
|
||||
|
||||
:::tip
|
||||
There are several other cache parameters that you can use to tune your storage, see [using local cache](/docs/en/operations/storing-data.md/#using-local-cache) for the details.
|
||||
:::
|
||||
|
||||
S3 disk can be configured as `main` or `cold` storage:
|
||||
``` xml
|
||||
<storage_configuration>
|
||||
|
||||
@ -1210,6 +1210,7 @@ SELECT * FROM json_each_row_nested
|
||||
- [input_format_json_read_objects_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `false`.
|
||||
- [input_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings-formats.md/#input_format_json_named_tuples_as_objects) - parse named tuple columns as JSON objects. Default value - `true`.
|
||||
- [input_format_json_defaults_for_missing_elements_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_defaults_for_missing_elements_in_named_tuple) - insert default values for missing elements in JSON object while parsing named tuple. Default value - `true`.
|
||||
- [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - Ignore unknown keys in json object for named tuples. Default value - `false`.
|
||||
- [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`.
|
||||
- [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`.
|
||||
- [output_format_json_quote_denormals](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`.
|
||||
|
||||
@ -239,7 +239,7 @@ Example of configuration:
|
||||
<clickhouse>
|
||||
<named_collections>
|
||||
<remote1>
|
||||
<host>localhost</host>
|
||||
<host>remote_host</host>
|
||||
<port>9000</port>
|
||||
<database>system</database>
|
||||
<user>foo</user>
|
||||
|
||||
@ -96,4 +96,4 @@ setting [query_result_cache_store_results_of_queries_with_nondeterministic_funct
|
||||
Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a
|
||||
row policy on a table by running the same query as another user B for whom no such policy exists. However, if necessary, cache entries can
|
||||
be marked accessible by other users (i.e. shared) by supplying setting
|
||||
[query_result_cache_share_between_users]{settings/settings.md#query-result-cache-share-between-users}.
|
||||
[query_result_cache_share_between_users](settings/settings.md#query-result-cache-share-between-users).
|
||||
|
||||
@ -500,6 +500,12 @@ Parse named tuple columns as JSON objects.
|
||||
|
||||
Enabled by default.
|
||||
|
||||
## input_format_json_ignore_unknown_keys_in_named_tuple {#input_format_json_ignore_unknown_keys_in_named_tuple}
|
||||
|
||||
Ignore unknown keys in json object for named tuples.
|
||||
|
||||
Disabled by default.
|
||||
|
||||
## input_format_json_defaults_for_missing_elements_in_named_tuple {#input_format_json_defaults_for_missing_elements_in_named_tuple}
|
||||
|
||||
Insert default values for missing elements in JSON object while parsing named tuple.
|
||||
|
||||
@ -19,11 +19,11 @@ Example:
|
||||
|
||||
``` sql
|
||||
SELECT maxMap(a, b)
|
||||
FROM values('a Array(Int32), b Array(Int64)', ([1, 2], [2, 2]), ([2, 3], [1, 1]))
|
||||
FROM values('a Array(Char), b Array(Int64)', (['x', 'y'], [2, 2]), (['y', 'z'], [3, 1]))
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─maxMap(a, b)──────┐
|
||||
│ ([1,2,3],[2,2,1]) │
|
||||
└───────────────────┘
|
||||
┌─maxMap(a, b)───────────┐
|
||||
│ [['x','y','z'],[2,3,1]]│
|
||||
└────────────────────────┘
|
||||
```
|
||||
|
||||
@ -6,3 +6,7 @@ sidebar_position: 4
|
||||
# sum
|
||||
|
||||
Calculates the sum. Only works for numbers.
|
||||
|
||||
```
|
||||
SELECT sum(salary) FROM employees;
|
||||
```
|
||||
|
||||
@ -28,15 +28,16 @@ Returns an array of the values with maximum approximate sum of weights.
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT topKWeighted(10)(number, number) FROM numbers(1000)
|
||||
SELECT topKWeighted(2)(k, w) FROM
|
||||
VALUES('k Char, w UInt64', ('y', 1), ('y', 1), ('x', 5), ('y', 1), ('z', 10))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─topKWeighted(10)(number, number)──────────┐
|
||||
│ [999,998,997,996,995,994,993,992,991,990] │
|
||||
└───────────────────────────────────────────┘
|
||||
┌─topKWeighted(2)(k, w)──┐
|
||||
│ ['z','x'] │
|
||||
└────────────────────────┘
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
@ -215,10 +215,10 @@ The two-argument form of `toDayOfWeek()` enables you to specify whether the week
|
||||
|
||||
| Mode | First day of week | Range |
|
||||
|------|-------------------|------------------------------------------------|
|
||||
| 0 | Monday | 1-7, Monday = 1, Tuesday = 2, ..., Sunday = 7 |
|
||||
| 1 | Monday | 0-6, Monday = 0, Tuesday = 1, ..., Sunday = 6 |
|
||||
| 2 | Sunday | 0-6, Sunday = 0, Monday = 1, ..., Saturday = 6 |
|
||||
| 3 | Sunday | 1-7, Sunday = 1, Monday = 2, ..., Saturday = 7 |
|
||||
| 0 | Monday | 1-7: Monday = 1, Tuesday = 2, ..., Sunday = 7 |
|
||||
| 1 | Monday | 0-6: Monday = 0, Tuesday = 1, ..., Sunday = 6 |
|
||||
| 2 | Sunday | 0-6: Sunday = 0, Monday = 1, ..., Saturday = 6 |
|
||||
| 3 | Sunday | 1-7: Sunday = 1, Monday = 2, ..., Saturday = 7 |
|
||||
|
||||
Alias: `DAYOFWEEK`.
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@ The following operations are available:
|
||||
|
||||
- `ALTER TABLE [db].table_name [ON CLUSTER cluster] ADD INDEX name expression TYPE type GRANULARITY value [FIRST|AFTER name]` - Adds index description to tables metadata.
|
||||
|
||||
- `ALTER TABLE [db].table_name [ON CLUSTER cluster] DROP INDEX name` - Removes index description from tables metadata and deletes index files from disk.
|
||||
- `ALTER TABLE [db].table_name [ON CLUSTER cluster] DROP INDEX name` - Removes index description from tables metadata and deletes index files from disk. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
- `ALTER TABLE [db.]table_name [ON CLUSTER cluster] MATERIALIZE INDEX name [IN PARTITION partition_name]` - Rebuilds the secondary index `name` for the specified `partition_name`. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations). If `IN PARTITION` part is omitted then it rebuilds the index for the whole table data.
|
||||
|
||||
|
||||
@ -293,7 +293,7 @@ These codecs are designed to make compression more effective by using specific f
|
||||
|
||||
#### Gorilla
|
||||
|
||||
`Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see Compressing Values in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf).
|
||||
`Gorilla` — Calculates XOR between current and previous floating point value and writes it in compact binary form. The smaller the difference between consecutive values is, i.e. the slower the values of the series changes, the better the compression rate. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see section 4.1 in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](https://doi.org/10.14778/2824032.2824078).
|
||||
|
||||
#### FPC
|
||||
|
||||
|
||||
@ -362,3 +362,15 @@ Allows to drop filesystem cache.
|
||||
```sql
|
||||
SYSTEM DROP FILESYSTEM CACHE
|
||||
```
|
||||
|
||||
### SYNC FILE CACHE
|
||||
|
||||
:::note
|
||||
It's too heavy and has potential for misuse.
|
||||
:::
|
||||
|
||||
Will do sync syscall.
|
||||
|
||||
```sql
|
||||
SYSTEM SYNC FILE CACHE
|
||||
```
|
||||
|
||||
@ -27,7 +27,7 @@ $ cat /etc/clickhouse-server/config.d/named_collections.xml
|
||||
|
||||
## Именованные соединения для доступа к S3
|
||||
|
||||
Описание параметров смотри [Табличная Функция S3](../sql-reference/table-functions/s3.md).
|
||||
Описание параметров смотрите [Табличная Функция S3](../sql-reference/table-functions/s3.md).
|
||||
|
||||
Пример конфигурации:
|
||||
```xml
|
||||
@ -75,7 +75,7 @@ SELECT * FROM s3_engine_table LIMIT 3;
|
||||
|
||||
## Пример использования именованных соединений с базой данных MySQL
|
||||
|
||||
Описание параметров смотри [mysql](../sql-reference/table-functions/mysql.md).
|
||||
Описание параметров смотрите [mysql](../sql-reference/table-functions/mysql.md).
|
||||
|
||||
Пример конфигурации:
|
||||
```xml
|
||||
@ -147,7 +147,7 @@ SELECT dictGet('dict', 'B', 2);
|
||||
|
||||
## Пример использования именованных соединений с базой данных PostgreSQL
|
||||
|
||||
Описание параметров смотри [postgresql](../sql-reference/table-functions/postgresql.md).
|
||||
Описание параметров смотрите [postgresql](../sql-reference/table-functions/postgresql.md).
|
||||
|
||||
Пример конфигурации:
|
||||
```xml
|
||||
@ -227,3 +227,58 @@ SELECT dictGet('dict', 'b', 2);
|
||||
│ two │
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
## Пример использования именованных соединений с удалённой базой данных Сlickhouse
|
||||
|
||||
Описание параметров смотрите [remote](../sql-reference/table-functions/remote.md).
|
||||
|
||||
Пример конфигурации:
|
||||
```xml
|
||||
<clickhouse>
|
||||
<named_collections>
|
||||
<remote1>
|
||||
<host>remote_host</host>
|
||||
<port>9000</port>
|
||||
<database>system</database>
|
||||
<user>foo</user>
|
||||
<password>secret</password>
|
||||
</remote1>
|
||||
</named_collections>
|
||||
</clickhouse>
|
||||
```
|
||||
|
||||
### Пример использования именованных соединений с табличной функцией remote/remoteSecure
|
||||
|
||||
```sql
|
||||
SELECT * FROM remote(remote1, table = one);
|
||||
┌─dummy─┐
|
||||
│ 0 │
|
||||
└───────┘
|
||||
|
||||
SELECT * FROM remote(remote1, database = merge(system, '^one'));
|
||||
┌─dummy─┐
|
||||
│ 0 │
|
||||
└───────┘
|
||||
|
||||
INSERT INTO FUNCTION remote(remote1, database = default, table = test) VALUES (1,'a');
|
||||
|
||||
SELECT * FROM remote(remote1, database = default, table = test);
|
||||
┌─a─┬─b─┐
|
||||
│ 1 │ a │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
### Пример использования именованных соединений с внешним словарем с источником удалённым сервером Clickhouse
|
||||
|
||||
```sql
|
||||
CREATE DICTIONARY dict(a Int64, b String)
|
||||
PRIMARY KEY a
|
||||
SOURCE(CLICKHOUSE(NAME remote1 TABLE test DB default))
|
||||
LIFETIME(MIN 1 MAX 2)
|
||||
LAYOUT(HASHED());
|
||||
|
||||
SELECT dictGet('dict', 'b', 1);
|
||||
┌─dictGet('dict', 'b', 1)─┐
|
||||
│ a │
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
@ -277,7 +277,7 @@ private:
|
||||
}
|
||||
|
||||
if (queries.empty())
|
||||
throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED);
|
||||
throw Exception(ErrorCodes::EMPTY_DATA_PASSED, "Empty list of queries.");
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -719,7 +719,7 @@ bool Client::processWithFuzzing(const String & full_query)
|
||||
// uniformity.
|
||||
// Surprisingly, this is a client exception, because we get the
|
||||
// server exception w/o throwing (see onReceiveException()).
|
||||
client_exception = std::make_unique<Exception>(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode());
|
||||
client_exception = std::make_unique<Exception>(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode());
|
||||
have_error = true;
|
||||
}
|
||||
|
||||
@ -854,7 +854,7 @@ bool Client::processWithFuzzing(const String & full_query)
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
client_exception = std::make_unique<Exception>(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode());
|
||||
client_exception = std::make_unique<Exception>(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode());
|
||||
have_error = true;
|
||||
}
|
||||
|
||||
|
||||
@ -165,9 +165,8 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
|
||||
/// should throw exception early and make exception message more readable.
|
||||
if (const auto * insert_query = res->as<ASTInsertQuery>(); insert_query && insert_query->data)
|
||||
{
|
||||
throw Exception(
|
||||
"Can't format ASTInsertQuery with data, since data will be lost",
|
||||
DB::ErrorCodes::INVALID_FORMAT_INSERT_QUERY_WITH_DATA);
|
||||
throw Exception(DB::ErrorCodes::INVALID_FORMAT_INSERT_QUERY_WITH_DATA,
|
||||
"Can't format ASTInsertQuery with data, since data will be lost");
|
||||
}
|
||||
if (!quiet)
|
||||
{
|
||||
|
||||
@ -196,7 +196,7 @@ void Keeper::createServer(const std::string & listen_host, const char * port_nam
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception{message, ErrorCodes::NETWORK_ERROR};
|
||||
throw Exception::createDeprecated(message, ErrorCodes::NETWORK_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -375,7 +375,7 @@ try
|
||||
if (effective_user_id == 0)
|
||||
{
|
||||
message += " Run under 'sudo -u " + data_owner + "'.";
|
||||
throw Exception(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA);
|
||||
throw Exception::createDeprecated(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -243,7 +243,6 @@ ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl(
|
||||
const ColumnRawPtrs & columns,
|
||||
bool cat_features_are_strings) const
|
||||
{
|
||||
std::string error_msg = "Error occurred while applying CatBoost model: ";
|
||||
size_t column_size = columns.front()->size();
|
||||
|
||||
auto result = ColumnFloat64::create(column_size * tree_count);
|
||||
@ -265,7 +264,8 @@ ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl(
|
||||
result_buf, column_size * tree_count))
|
||||
{
|
||||
|
||||
throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL);
|
||||
throw Exception(ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL,
|
||||
"Error occurred while applying CatBoost model: {}", api.GetErrorString());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -288,7 +288,8 @@ ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl(
|
||||
cat_features_buf, cat_features_count,
|
||||
result_buf, column_size * tree_count))
|
||||
{
|
||||
throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL);
|
||||
throw Exception(ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL,
|
||||
"Error occurred while applying CatBoost model: {}", api.GetErrorString());
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -304,7 +305,8 @@ ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl(
|
||||
cat_features_buf, cat_features_count,
|
||||
result_buf, column_size * tree_count))
|
||||
{
|
||||
throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL);
|
||||
throw Exception(ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL,
|
||||
"Error occurred while applying CatBoost model: {}", api.GetErrorString());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -416,7 +416,7 @@ void Server::createServer(
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception{message, ErrorCodes::NETWORK_ERROR};
|
||||
throw Exception::createDeprecated(message, ErrorCodes::NETWORK_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -946,7 +946,7 @@ try
|
||||
if (effective_user_id == 0)
|
||||
{
|
||||
message += " Run under 'sudo -u " + data_owner + "'.";
|
||||
throw Exception(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA);
|
||||
throw Exception::createDeprecated(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -171,6 +171,7 @@ enum class AccessType
|
||||
M(SYSTEM_WAIT_LOADING_PARTS, "WAIT LOADING PARTS", TABLE, SYSTEM) \
|
||||
M(SYSTEM_SYNC_DATABASE_REPLICA, "SYNC DATABASE REPLICA", DATABASE, SYSTEM) \
|
||||
M(SYSTEM_SYNC_TRANSACTION_LOG, "SYNC TRANSACTION LOG", GLOBAL, SYSTEM) \
|
||||
M(SYSTEM_SYNC_FILE_CACHE, "SYNC FILE CACHE", GLOBAL, SYSTEM) \
|
||||
M(SYSTEM_FLUSH_DISTRIBUTED, "FLUSH DISTRIBUTED", TABLE, SYSTEM_FLUSH) \
|
||||
M(SYSTEM_FLUSH_LOGS, "FLUSH LOGS", GLOBAL, SYSTEM_FLUSH) \
|
||||
M(SYSTEM_FLUSH, "", GROUP, SYSTEM) \
|
||||
|
||||
@ -79,9 +79,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_vi
|
||||
::DB::encodeSHA256(text, hash.data());
|
||||
return hash;
|
||||
#else
|
||||
throw DB::Exception(
|
||||
"SHA256 passwords support is disabled, because ClickHouse was built without SSL library",
|
||||
DB::ErrorCodes::SUPPORT_IS_DISABLED);
|
||||
throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, "SHA256 passwords support is disabled, because ClickHouse was built without SSL library");
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -484,13 +484,15 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
return true;
|
||||
};
|
||||
|
||||
auto access_denied = [&](const String & error_msg, int error_code [[maybe_unused]])
|
||||
auto access_denied = [&]<typename... FmtArgs>(int error_code [[maybe_unused]],
|
||||
FormatStringHelper<String, FmtArgs...> fmt_string [[maybe_unused]],
|
||||
FmtArgs && ...fmt_args [[maybe_unused]])
|
||||
{
|
||||
if (trace_log)
|
||||
LOG_TRACE(trace_log, "Access denied: {}{}", (AccessRightsElement{flags, args...}.toStringWithoutOptions()),
|
||||
(grant_option ? " WITH GRANT OPTION" : ""));
|
||||
if constexpr (throw_if_denied)
|
||||
throw Exception(getUserName() + ": " + error_msg, error_code);
|
||||
throw Exception(error_code, std::move(fmt_string), getUserName(), std::forward<FmtArgs>(fmt_args)...);
|
||||
return false;
|
||||
};
|
||||
|
||||
@ -519,18 +521,16 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
{
|
||||
if (grant_option && acs->isGranted(flags, args...))
|
||||
{
|
||||
return access_denied(
|
||||
"Not enough privileges. "
|
||||
return access_denied(ErrorCodes::ACCESS_DENIED,
|
||||
"{}: Not enough privileges. "
|
||||
"The required privileges have been granted, but without grant option. "
|
||||
"To execute this query it's necessary to have grant "
|
||||
+ AccessRightsElement{flags, args...}.toStringWithoutOptions() + " WITH GRANT OPTION",
|
||||
ErrorCodes::ACCESS_DENIED);
|
||||
"To execute this query it's necessary to have grant {} WITH GRANT OPTION",
|
||||
AccessRightsElement{flags, args...}.toStringWithoutOptions());
|
||||
}
|
||||
|
||||
return access_denied(
|
||||
"Not enough privileges. To execute this query it's necessary to have grant "
|
||||
+ AccessRightsElement{flags, args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""),
|
||||
ErrorCodes::ACCESS_DENIED);
|
||||
return access_denied(ErrorCodes::ACCESS_DENIED,
|
||||
"{}: Not enough privileges. To execute this query it's necessary to have grant {}",
|
||||
AccessRightsElement{flags, args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""));
|
||||
}
|
||||
|
||||
struct PrecalculatedFlags
|
||||
@ -557,32 +557,34 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
if (params.readonly)
|
||||
{
|
||||
if constexpr (grant_option)
|
||||
return access_denied("Cannot change grants in readonly mode.", ErrorCodes::READONLY);
|
||||
return access_denied(ErrorCodes::READONLY, "{}: Cannot change grants in readonly mode.");
|
||||
if ((flags & precalc.not_readonly_flags) ||
|
||||
((params.readonly == 1) && (flags & precalc.not_readonly_1_flags)))
|
||||
{
|
||||
if (params.interface == ClientInfo::Interface::HTTP && params.http_method == ClientInfo::HTTPMethod::GET)
|
||||
{
|
||||
return access_denied(
|
||||
"Cannot execute query in readonly mode. "
|
||||
"For queries over HTTP, method GET implies readonly. You should use method POST for modifying queries",
|
||||
ErrorCodes::READONLY);
|
||||
return access_denied(ErrorCodes::READONLY,
|
||||
"{}: Cannot execute query in readonly mode. "
|
||||
"For queries over HTTP, method GET implies readonly. "
|
||||
"You should use method POST for modifying queries");
|
||||
}
|
||||
else
|
||||
return access_denied("Cannot execute query in readonly mode", ErrorCodes::READONLY);
|
||||
return access_denied(ErrorCodes::READONLY, "{}: Cannot execute query in readonly mode");
|
||||
}
|
||||
}
|
||||
|
||||
if (!params.allow_ddl && !grant_option)
|
||||
{
|
||||
if (flags & precalc.ddl_flags)
|
||||
return access_denied("Cannot execute query. DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
|
||||
return access_denied(ErrorCodes::QUERY_IS_PROHIBITED,
|
||||
"Cannot execute query. DDL queries are prohibited for the user {}");
|
||||
}
|
||||
|
||||
if (!params.allow_introspection && !grant_option)
|
||||
{
|
||||
if (flags & precalc.introspection_flags)
|
||||
return access_denied("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED);
|
||||
return access_denied(ErrorCodes::FUNCTION_NOT_ALLOWED, "{}: Introspection functions are disabled, "
|
||||
"because setting 'allow_introspection_functions' is set to 0");
|
||||
}
|
||||
|
||||
return access_granted();
|
||||
@ -679,11 +681,13 @@ void ContextAccess::checkGrantOption(const AccessRightsElements & elements) cons
|
||||
template <bool throw_if_denied, typename Container, typename GetNameFunction>
|
||||
bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const
|
||||
{
|
||||
auto show_error = [this](const String & msg, int error_code [[maybe_unused]])
|
||||
auto show_error = []<typename... FmtArgs>(int error_code [[maybe_unused]],
|
||||
FormatStringHelper<FmtArgs...> fmt_string [[maybe_unused]],
|
||||
FmtArgs && ...fmt_args [[maybe_unused]])
|
||||
{
|
||||
UNUSED(this);
|
||||
if constexpr (throw_if_denied)
|
||||
throw Exception(getUserName() + ": " + msg, error_code);
|
||||
throw Exception(error_code, std::move(fmt_string), std::forward<FmtArgs>(fmt_args)...);
|
||||
return false;
|
||||
};
|
||||
|
||||
if (is_full_access)
|
||||
@ -691,7 +695,7 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
|
||||
|
||||
if (user_was_dropped)
|
||||
{
|
||||
show_error("User has been dropped", ErrorCodes::UNKNOWN_USER);
|
||||
show_error(ErrorCodes::UNKNOWN_USER, "User has been dropped");
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -716,14 +720,15 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
|
||||
role_name = "ID {" + toString(role_id) + "}";
|
||||
|
||||
if (info->enabled_roles.count(role_id))
|
||||
show_error("Not enough privileges. "
|
||||
"Role " + backQuote(*role_name) + " is granted, but without ADMIN option. "
|
||||
"To execute this query it's necessary to have the role " + backQuoteIfNeed(*role_name) + " granted with ADMIN option.",
|
||||
ErrorCodes::ACCESS_DENIED);
|
||||
show_error(ErrorCodes::ACCESS_DENIED,
|
||||
"Not enough privileges. "
|
||||
"Role {} is granted, but without ADMIN option. "
|
||||
"To execute this query it's necessary to have the role {} granted with ADMIN option.",
|
||||
backQuote(*role_name), backQuoteIfNeed(*role_name));
|
||||
else
|
||||
show_error("Not enough privileges. "
|
||||
"To execute this query it's necessary to have the role " + backQuoteIfNeed(*role_name) + " granted with ADMIN option.",
|
||||
ErrorCodes::ACCESS_DENIED);
|
||||
show_error(ErrorCodes::ACCESS_DENIED, "Not enough privileges. "
|
||||
"To execute this query it's necessary to have the role {} granted with ADMIN option.",
|
||||
backQuoteIfNeed(*role_name));
|
||||
}
|
||||
|
||||
return false;
|
||||
|
||||
@ -81,7 +81,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co
|
||||
{
|
||||
ret = krb5_cc_resolve(k5.ctx, cache_name.c_str(), &k5.out_cc);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in resolving cache{}", fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in resolving cache: {}", fmtError(ret));
|
||||
LOG_TRACE(log,"Resolved cache");
|
||||
}
|
||||
else
|
||||
@ -89,7 +89,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co
|
||||
// Resolve the default cache and get its type and default principal (if it is initialized).
|
||||
ret = krb5_cc_default(k5.ctx, &defcache);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while getting default cache{}", fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while getting default cache: {}", fmtError(ret));
|
||||
LOG_TRACE(log,"Resolved default cache");
|
||||
deftype = krb5_cc_get_type(k5.ctx, defcache);
|
||||
if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0)
|
||||
@ -99,7 +99,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co
|
||||
// Use the specified principal name.
|
||||
ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), 0, &k5.me);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when parsing principal name {}", principal + fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when parsing principal name ({}): {}", principal, fmtError(ret));
|
||||
|
||||
// Cache related commands
|
||||
if (k5.out_cc == nullptr && krb5_cc_support_switch(k5.ctx, deftype))
|
||||
@ -107,7 +107,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co
|
||||
// Use an existing cache for the client principal if we can.
|
||||
ret = krb5_cc_cache_match(k5.ctx, k5.me, &k5.out_cc);
|
||||
if (ret && ret != KRB5_CC_NOTFOUND)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while searching for cache for {}", principal + fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while searching for cache for ({}): {}", principal, fmtError(ret));
|
||||
if (0 == ret)
|
||||
{
|
||||
LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc));
|
||||
@ -118,7 +118,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co
|
||||
// Create a new cache to avoid overwriting the initialized default cache.
|
||||
ret = krb5_cc_new_unique(k5.ctx, deftype, nullptr, &k5.out_cc);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while generating new cache{}", fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while generating new cache: {}", fmtError(ret));
|
||||
LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc));
|
||||
k5.switch_to_cache = 1;
|
||||
}
|
||||
@ -134,24 +134,24 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co
|
||||
|
||||
ret = krb5_unparse_name(k5.ctx, k5.me, &k5.name);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when unparsing name{}", fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when unparsing name: {}", fmtError(ret));
|
||||
LOG_TRACE(log,"Using principal: {}", k5.name);
|
||||
|
||||
// Allocate a new initial credential options structure.
|
||||
ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in options allocation{}", fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in options allocation: {}", fmtError(ret));
|
||||
|
||||
// Resolve keytab
|
||||
ret = krb5_kt_resolve(k5.ctx, keytab_file.c_str(), &keytab);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in resolving keytab {}{}", keytab_file, fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in resolving keytab ({}): {}", keytab_file, fmtError(ret));
|
||||
LOG_TRACE(log,"Using keytab: {}", keytab_file);
|
||||
|
||||
// Set an output credential cache in initial credential options.
|
||||
ret = krb5_get_init_creds_opt_set_out_ccache(k5.ctx, options, k5.out_cc);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in setting output credential cache{}", fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in setting output credential cache: {}", fmtError(ret));
|
||||
|
||||
// Action: init or renew
|
||||
LOG_TRACE(log,"Trying to renew credentials");
|
||||
@ -165,7 +165,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co
|
||||
// Request KDC for an initial credentials using keytab.
|
||||
ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in getting initial credentials{}", fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in getting initial credentials: {}", fmtError(ret));
|
||||
else
|
||||
LOG_TRACE(log,"Got initial credentials");
|
||||
}
|
||||
@ -175,7 +175,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co
|
||||
// Initialize a credential cache. Destroy any existing contents of cache and initialize it for the default principal.
|
||||
ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when initializing cache{}", fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when initializing cache: {}", fmtError(ret));
|
||||
LOG_TRACE(log,"Initialized cache");
|
||||
// Store credentials in a credential cache.
|
||||
ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds);
|
||||
@ -189,7 +189,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co
|
||||
// Make a credential cache the primary cache for its collection.
|
||||
ret = krb5_cc_switch(k5.ctx, k5.out_cc);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while switching to new cache{}", fmtError(ret));
|
||||
throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while switching to new cache: {}", fmtError(ret));
|
||||
}
|
||||
|
||||
LOG_TRACE(log,"Authenticated to Kerberos v5");
|
||||
|
||||
@ -205,7 +205,7 @@ void LDAPClient::handleError(int result_code, String text)
|
||||
}
|
||||
}
|
||||
|
||||
throw Exception(text, ErrorCodes::LDAP_ERROR);
|
||||
throw Exception::createDeprecated(text, ErrorCodes::LDAP_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
@ -569,7 +569,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
|
||||
message += matched_msg;
|
||||
}
|
||||
|
||||
throw Exception(message, ErrorCodes::LDAP_ERROR);
|
||||
throw Exception::createDeprecated(message, ErrorCodes::LDAP_ERROR);
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
@ -266,7 +266,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, const Field & n
|
||||
if (!explain.empty())
|
||||
{
|
||||
if (reaction == THROW_ON_VIOLATION)
|
||||
throw Exception(explain, code);
|
||||
throw Exception::createDeprecated(explain, code);
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -106,7 +106,7 @@ public:
|
||||
default:
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Map key type " + key_type->getName() + " is not is not supported by combinator " + getName());
|
||||
"Map key type {} is not is not supported by combinator {}", key_type->getName(), getName());
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
@ -66,13 +66,13 @@ public:
|
||||
, kind(kind_)
|
||||
{
|
||||
if (!isNativeNumber(arguments[0]))
|
||||
throw Exception{getName() + ": first argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{}: first argument must be represented by integer", getName());
|
||||
|
||||
if (!isNativeNumber(arguments[1]))
|
||||
throw Exception{getName() + ": second argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{}: second argument must be represented by integer", getName());
|
||||
|
||||
if (!arguments[0]->equals(*arguments[1]))
|
||||
throw Exception{getName() + ": arguments must have the same type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{}: arguments must have the same type", getName());
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
|
||||
@ -88,9 +88,9 @@ createAggregateFunctionSequenceNode(const std::string & name, const DataTypes &
|
||||
name, toString(min_required_args + 1));
|
||||
|
||||
if (argument_types.size() > max_events_size + min_required_args)
|
||||
throw Exception(fmt::format(
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||
"Aggregate function '{}' requires at most {} (timestamp, value_column, ...{} events) arguments.",
|
||||
name, max_events_size + min_required_args, max_events_size), ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
name, max_events_size + min_required_args, max_events_size);
|
||||
|
||||
if (const auto * cond_arg = argument_types[2].get(); cond_arg && !isUInt8(cond_arg))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of third argument of aggregate function {}, "
|
||||
@ -100,9 +100,8 @@ createAggregateFunctionSequenceNode(const std::string & name, const DataTypes &
|
||||
{
|
||||
const auto * cond_arg = argument_types[i].get();
|
||||
if (!isUInt8(cond_arg))
|
||||
throw Exception(fmt::format(
|
||||
"Illegal type '{}' of {} argument of aggregate function '{}', must be UInt8", cond_arg->getName(), i + 1, name),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type '{}' of {} argument of aggregate function '{}', must be UInt8", cond_arg->getName(), i + 1, name);
|
||||
}
|
||||
|
||||
if (WhichDataType(argument_types[1].get()).idx != TypeIndex::String)
|
||||
|
||||
@ -235,7 +235,7 @@ private:
|
||||
if (skip_degree_ == skip_degree)
|
||||
return;
|
||||
if (skip_degree_ > detail::MAX_SKIP_DEGREE)
|
||||
throw DB::Exception{"skip_degree exceeds maximum value", DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED};
|
||||
throw DB::Exception(DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, "skip_degree exceeds maximum value");
|
||||
skip_degree = skip_degree_;
|
||||
if (skip_degree == detail::MAX_SKIP_DEGREE)
|
||||
skip_mask = static_cast<UInt32>(-1);
|
||||
|
||||
@ -1,14 +1,15 @@
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
#include <Common/SettingsChanges.h>
|
||||
#include <base/scope_guard.h>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Core/Settings.h>
|
||||
|
||||
#include <Analyzer/IQueryTreeNode.h>
|
||||
#include <Analyzer/QueryNode.h>
|
||||
#include <Analyzer/UnionNode.h>
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -89,4 +90,134 @@ private:
|
||||
template <typename Derived>
|
||||
using ConstInDepthQueryTreeVisitor = InDepthQueryTreeVisitor<Derived, true /*const_visitor*/>;
|
||||
|
||||
/** Same as InDepthQueryTreeVisitor and additionally keeps track of current scope context.
|
||||
* This can be useful if your visitor has special logic that depends on current scope context.
|
||||
*/
|
||||
template <typename Derived, bool const_visitor = false>
|
||||
class InDepthQueryTreeVisitorWithContext
|
||||
{
|
||||
public:
|
||||
using VisitQueryTreeNodeType = std::conditional_t<const_visitor, const QueryTreeNodePtr, QueryTreeNodePtr>;
|
||||
|
||||
explicit InDepthQueryTreeVisitorWithContext(ContextPtr context)
|
||||
: current_context(std::move(context))
|
||||
{}
|
||||
|
||||
/// Return true if visitor should traverse tree top to bottom, false otherwise
|
||||
bool shouldTraverseTopToBottom() const
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Return true if visitor should visit child, false otherwise
|
||||
bool needChildVisit(VisitQueryTreeNodeType & parent [[maybe_unused]], VisitQueryTreeNodeType & child [[maybe_unused]])
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
const ContextPtr & getContext() const
|
||||
{
|
||||
return current_context;
|
||||
}
|
||||
|
||||
const Settings & getSettings() const
|
||||
{
|
||||
return current_context->getSettingsRef();
|
||||
}
|
||||
|
||||
void visit(VisitQueryTreeNodeType & query_tree_node)
|
||||
{
|
||||
auto current_scope_context_ptr = current_context;
|
||||
SCOPE_EXIT(
|
||||
current_context = std::move(current_scope_context_ptr);
|
||||
);
|
||||
|
||||
if (auto * query_node = query_tree_node->template as<QueryNode>())
|
||||
current_context = query_node->getContext();
|
||||
else if (auto * union_node = query_tree_node->template as<UnionNode>())
|
||||
current_context = union_node->getContext();
|
||||
|
||||
bool traverse_top_to_bottom = getDerived().shouldTraverseTopToBottom();
|
||||
if (!traverse_top_to_bottom)
|
||||
visitChildren(query_tree_node);
|
||||
|
||||
getDerived().visitImpl(query_tree_node);
|
||||
|
||||
if (traverse_top_to_bottom)
|
||||
visitChildren(query_tree_node);
|
||||
}
|
||||
private:
|
||||
Derived & getDerived()
|
||||
{
|
||||
return *static_cast<Derived *>(this);
|
||||
}
|
||||
|
||||
const Derived & getDerived() const
|
||||
{
|
||||
return *static_cast<Derived *>(this);
|
||||
}
|
||||
|
||||
void visitChildren(VisitQueryTreeNodeType & expression)
|
||||
{
|
||||
for (auto & child : expression->getChildren())
|
||||
{
|
||||
if (!child)
|
||||
continue;
|
||||
|
||||
bool need_visit_child = getDerived().needChildVisit(expression, child);
|
||||
|
||||
if (need_visit_child)
|
||||
visit(child);
|
||||
}
|
||||
}
|
||||
|
||||
ContextPtr current_context;
|
||||
};
|
||||
|
||||
template <typename Derived>
|
||||
using ConstInDepthQueryTreeVisitorWithContext = InDepthQueryTreeVisitorWithContext<Derived, true /*const_visitor*/>;
|
||||
|
||||
/** Visitor that use another visitor to visit node only if condition for visiting node is true.
|
||||
* For example, your visitor need to visit only query tree nodes or union nodes.
|
||||
*
|
||||
* Condition interface:
|
||||
* struct Condition
|
||||
* {
|
||||
* bool operator()(VisitQueryTreeNodeType & node)
|
||||
* {
|
||||
* return shouldNestedVisitorVisitNode(node);
|
||||
* }
|
||||
* }
|
||||
*/
|
||||
template <typename Visitor, typename Condition, bool const_visitor = false>
|
||||
class InDepthQueryTreeConditionalVisitor : public InDepthQueryTreeVisitor<InDepthQueryTreeConditionalVisitor<Visitor, Condition, const_visitor>, const_visitor>
|
||||
{
|
||||
public:
|
||||
using Base = InDepthQueryTreeVisitor<InDepthQueryTreeConditionalVisitor<Visitor, Condition, const_visitor>, const_visitor>;
|
||||
using VisitQueryTreeNodeType = typename Base::VisitQueryTreeNodeType;
|
||||
|
||||
explicit InDepthQueryTreeConditionalVisitor(Visitor & visitor_, Condition & condition_)
|
||||
: visitor(visitor_)
|
||||
, condition(condition_)
|
||||
{
|
||||
}
|
||||
|
||||
bool shouldTraverseTopToBottom() const
|
||||
{
|
||||
return visitor.shouldTraverseTopToBottom();
|
||||
}
|
||||
|
||||
void visitImpl(VisitQueryTreeNodeType & query_tree_node)
|
||||
{
|
||||
if (condition(query_tree_node))
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
Visitor & visitor;
|
||||
Condition & condition;
|
||||
};
|
||||
|
||||
template <typename Visitor, typename Condition>
|
||||
using ConstInDepthQueryTreeConditionalVisitor = InDepthQueryTreeConditionalVisitor<Visitor, Condition, true /*const_visitor*/>;
|
||||
|
||||
}
|
||||
|
||||
@ -45,12 +45,11 @@ Field zeroField(const Field & value)
|
||||
* TODO: Support `groupBitAnd`, `groupBitOr`, `groupBitXor` functions.
|
||||
* TODO: Support rewrite `f((2 * n) * n)` into '2 * f(n * n)'.
|
||||
*/
|
||||
class AggregateFunctionsArithmericOperationsVisitor : public InDepthQueryTreeVisitor<AggregateFunctionsArithmericOperationsVisitor>
|
||||
class AggregateFunctionsArithmericOperationsVisitor : public InDepthQueryTreeVisitorWithContext<AggregateFunctionsArithmericOperationsVisitor>
|
||||
{
|
||||
public:
|
||||
explicit AggregateFunctionsArithmericOperationsVisitor(ContextPtr context_)
|
||||
: context(std::move(context_))
|
||||
{}
|
||||
using Base = InDepthQueryTreeVisitorWithContext<AggregateFunctionsArithmericOperationsVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
/// Traverse tree bottom to top
|
||||
static bool shouldTraverseTopToBottom()
|
||||
@ -60,6 +59,9 @@ public:
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_arithmetic_operations_in_aggregate_functions)
|
||||
return;
|
||||
|
||||
auto * aggregate_function_node = node->as<FunctionNode>();
|
||||
if (!aggregate_function_node || !aggregate_function_node->isAggregateFunction())
|
||||
return;
|
||||
@ -175,7 +177,7 @@ private:
|
||||
|
||||
inline void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const
|
||||
{
|
||||
auto function = FunctionFactory::instance().get(function_name, context);
|
||||
auto function = FunctionFactory::instance().get(function_name, getContext());
|
||||
function_node.resolveAsFunction(function->build(function_node.getArgumentColumns()));
|
||||
}
|
||||
|
||||
@ -191,8 +193,6 @@ private:
|
||||
|
||||
function_node.resolveAsAggregateFunction(std::move(aggregate_function));
|
||||
}
|
||||
|
||||
ContextPtr context;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1,17 +1,23 @@
|
||||
#include <Analyzer/Passes/ConvertOrLikeChainPass.h>
|
||||
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
#include <Analyzer/Passes/ConvertOrLikeChainPass.h>
|
||||
|
||||
#include <Core/Field.h>
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/likePatternToRegexp.h>
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
#include <Analyzer/ConstantNode.h>
|
||||
#include <Analyzer/UnionNode.h>
|
||||
#include <Analyzer/FunctionNode.h>
|
||||
#include <Analyzer/HashUtils.h>
|
||||
#include <Analyzer/InDepthQueryTreeVisitor.h>
|
||||
#include <Core/Field.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/likePatternToRegexp.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -19,36 +25,28 @@ namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
class ConvertOrLikeChainVisitor : public InDepthQueryTreeVisitor<ConvertOrLikeChainVisitor>
|
||||
class ConvertOrLikeChainVisitor : public InDepthQueryTreeVisitorWithContext<ConvertOrLikeChainVisitor>
|
||||
{
|
||||
using FunctionNodes = std::vector<std::shared_ptr<FunctionNode>>;
|
||||
|
||||
const FunctionOverloadResolverPtr match_function_ref;
|
||||
const FunctionOverloadResolverPtr or_function_resolver;
|
||||
public:
|
||||
using Base = InDepthQueryTreeVisitorWithContext<ConvertOrLikeChainVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
explicit ConvertOrLikeChainVisitor(ContextPtr context)
|
||||
: InDepthQueryTreeVisitor<ConvertOrLikeChainVisitor>()
|
||||
, match_function_ref(FunctionFactory::instance().get("multiMatchAny", context))
|
||||
, or_function_resolver(FunctionFactory::instance().get("or", context))
|
||||
explicit ConvertOrLikeChainVisitor(FunctionOverloadResolverPtr or_function_resolver_,
|
||||
FunctionOverloadResolverPtr match_function_resolver_,
|
||||
ContextPtr context)
|
||||
: Base(std::move(context))
|
||||
, or_function_resolver(std::move(or_function_resolver_))
|
||||
, match_function_resolver(std::move(match_function_resolver_))
|
||||
{}
|
||||
|
||||
static bool needChildVisit(VisitQueryTreeNodeType & parent, VisitQueryTreeNodeType &)
|
||||
bool needChildVisit(VisitQueryTreeNodeType &, VisitQueryTreeNodeType &)
|
||||
{
|
||||
ContextPtr context;
|
||||
if (auto * query = parent->as<QueryNode>())
|
||||
context = query->getContext();
|
||||
else if (auto * union_node = parent->as<UnionNode>())
|
||||
context = union_node->getContext();
|
||||
if (context)
|
||||
{
|
||||
const auto & settings = context->getSettingsRef();
|
||||
return settings.optimize_or_like_chain
|
||||
&& settings.allow_hyperscan
|
||||
&& settings.max_hyperscan_regexp_length == 0
|
||||
&& settings.max_hyperscan_regexp_total_length == 0;
|
||||
}
|
||||
return true;
|
||||
const auto & settings = getSettings();
|
||||
|
||||
return settings.optimize_or_like_chain
|
||||
&& settings.allow_hyperscan
|
||||
&& settings.max_hyperscan_regexp_length == 0
|
||||
&& settings.max_hyperscan_regexp_total_length == 0;
|
||||
}
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
@ -61,27 +59,28 @@ public:
|
||||
|
||||
QueryTreeNodePtrWithHashMap<Array> node_to_patterns;
|
||||
FunctionNodes match_functions;
|
||||
for (auto & arg : function_node->getArguments())
|
||||
{
|
||||
unique_elems.push_back(arg);
|
||||
|
||||
auto * arg_func = arg->as<FunctionNode>();
|
||||
if (!arg_func)
|
||||
for (auto & argument : function_node->getArguments())
|
||||
{
|
||||
unique_elems.push_back(argument);
|
||||
|
||||
auto * argument_function = argument->as<FunctionNode>();
|
||||
if (!argument_function)
|
||||
continue;
|
||||
|
||||
const bool is_like = arg_func->getFunctionName() == "like";
|
||||
const bool is_ilike = arg_func->getFunctionName() == "ilike";
|
||||
const bool is_like = argument_function->getFunctionName() == "like";
|
||||
const bool is_ilike = argument_function->getFunctionName() == "ilike";
|
||||
|
||||
/// Not {i}like -> bail out.
|
||||
if (!is_like && !is_ilike)
|
||||
continue;
|
||||
|
||||
const auto & like_arguments = arg_func->getArguments().getNodes();
|
||||
const auto & like_arguments = argument_function->getArguments().getNodes();
|
||||
if (like_arguments.size() != 2)
|
||||
continue;
|
||||
|
||||
auto identifier = like_arguments[0];
|
||||
auto * pattern = like_arguments[1]->as<ConstantNode>();
|
||||
const auto & like_first_argument = like_arguments[0];
|
||||
const auto * pattern = like_arguments[1]->as<ConstantNode>();
|
||||
if (!pattern || !isString(pattern->getResultType()))
|
||||
continue;
|
||||
|
||||
@ -91,17 +90,20 @@ public:
|
||||
regexp = "(?i)" + regexp;
|
||||
|
||||
unique_elems.pop_back();
|
||||
auto it = node_to_patterns.find(identifier);
|
||||
|
||||
auto it = node_to_patterns.find(like_first_argument);
|
||||
if (it == node_to_patterns.end())
|
||||
{
|
||||
it = node_to_patterns.insert({identifier, Array{}}).first;
|
||||
it = node_to_patterns.insert({like_first_argument, Array{}}).first;
|
||||
|
||||
/// The second argument will be added when all patterns are known.
|
||||
auto match_function = std::make_shared<FunctionNode>("multiMatchAny");
|
||||
match_function->getArguments().getNodes().push_back(identifier);
|
||||
|
||||
match_function->getArguments().getNodes().push_back(like_first_argument);
|
||||
match_functions.push_back(match_function);
|
||||
|
||||
unique_elems.push_back(std::move(match_function));
|
||||
}
|
||||
|
||||
it->second.push_back(regexp);
|
||||
}
|
||||
|
||||
@ -111,23 +113,29 @@ public:
|
||||
auto & arguments = match_function->getArguments().getNodes();
|
||||
auto & patterns = node_to_patterns.at(arguments[0]);
|
||||
arguments.push_back(std::make_shared<ConstantNode>(Field{std::move(patterns)}));
|
||||
match_function->resolveAsFunction(match_function_ref);
|
||||
match_function->resolveAsFunction(match_function_resolver);
|
||||
}
|
||||
|
||||
/// OR must have at least two arguments.
|
||||
if (unique_elems.size() == 1)
|
||||
unique_elems.push_back(std::make_shared<ConstantNode>(false));
|
||||
unique_elems.push_back(std::make_shared<ConstantNode>(static_cast<UInt8>(0)));
|
||||
|
||||
function_node->getArguments().getNodes() = std::move(unique_elems);
|
||||
function_node->resolveAsFunction(or_function_resolver);
|
||||
}
|
||||
private:
|
||||
using FunctionNodes = std::vector<std::shared_ptr<FunctionNode>>;
|
||||
const FunctionOverloadResolverPtr or_function_resolver;
|
||||
const FunctionOverloadResolverPtr match_function_resolver;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void ConvertOrLikeChainPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
void ConvertOrLikeChainPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
ConvertOrLikeChainVisitor visitor(context);
|
||||
auto or_function_resolver = FunctionFactory::instance().get("or", context);
|
||||
auto match_function_resolver = FunctionFactory::instance().get("multiMatchAny", context);
|
||||
ConvertOrLikeChainVisitor visitor(std::move(or_function_resolver), std::move(match_function_resolver), std::move(context));
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
|
||||
@ -16,11 +16,17 @@ namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
class CountDistinctVisitor : public InDepthQueryTreeVisitor<CountDistinctVisitor>
|
||||
class CountDistinctVisitor : public InDepthQueryTreeVisitorWithContext<CountDistinctVisitor>
|
||||
{
|
||||
public:
|
||||
static void visitImpl(QueryTreeNodePtr & node)
|
||||
using Base = InDepthQueryTreeVisitorWithContext<CountDistinctVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().count_distinct_optimization)
|
||||
return;
|
||||
|
||||
auto * query_node = node->as<QueryNode>();
|
||||
|
||||
/// Check that query has only SELECT clause
|
||||
@ -78,9 +84,9 @@ public:
|
||||
|
||||
}
|
||||
|
||||
void CountDistinctPass::run(QueryTreeNodePtr query_tree_node, ContextPtr)
|
||||
void CountDistinctPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
CountDistinctVisitor visitor;
|
||||
CountDistinctVisitor visitor(std::move(context));
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
|
||||
@ -16,12 +16,11 @@ namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
class CustomizeFunctionsVisitor : public InDepthQueryTreeVisitor<CustomizeFunctionsVisitor>
|
||||
class CustomizeFunctionsVisitor : public InDepthQueryTreeVisitorWithContext<CustomizeFunctionsVisitor>
|
||||
{
|
||||
public:
|
||||
explicit CustomizeFunctionsVisitor(ContextPtr & context_)
|
||||
: context(context_)
|
||||
{}
|
||||
using Base = InDepthQueryTreeVisitorWithContext<CustomizeFunctionsVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node) const
|
||||
{
|
||||
@ -29,7 +28,7 @@ public:
|
||||
if (!function_node)
|
||||
return;
|
||||
|
||||
const auto & settings = context->getSettingsRef();
|
||||
const auto & settings = getSettings();
|
||||
|
||||
/// After successful function replacement function name and function name lowercase must be recalculated
|
||||
auto function_name = function_node->getFunctionName();
|
||||
@ -154,19 +153,16 @@ public:
|
||||
|
||||
inline void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const
|
||||
{
|
||||
auto function = FunctionFactory::instance().get(function_name, context);
|
||||
auto function = FunctionFactory::instance().get(function_name, getContext());
|
||||
function_node.resolveAsFunction(function->build(function_node.getArgumentColumns()));
|
||||
}
|
||||
|
||||
private:
|
||||
ContextPtr & context;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void CustomizeFunctionsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
CustomizeFunctionsVisitor visitor(context);
|
||||
CustomizeFunctionsVisitor visitor(std::move(context));
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
|
||||
@ -22,15 +22,17 @@ namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
class FunctionToSubcolumnsVisitor : public InDepthQueryTreeVisitor<FunctionToSubcolumnsVisitor>
|
||||
class FunctionToSubcolumnsVisitor : public InDepthQueryTreeVisitorWithContext<FunctionToSubcolumnsVisitor>
|
||||
{
|
||||
public:
|
||||
explicit FunctionToSubcolumnsVisitor(ContextPtr & context_)
|
||||
: context(context_)
|
||||
{}
|
||||
using Base = InDepthQueryTreeVisitorWithContext<FunctionToSubcolumnsVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node) const
|
||||
{
|
||||
if (!getSettings().optimize_functions_to_subcolumns)
|
||||
return;
|
||||
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
if (!function_node)
|
||||
return;
|
||||
@ -192,11 +194,9 @@ public:
|
||||
private:
|
||||
inline void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const
|
||||
{
|
||||
auto function = FunctionFactory::instance().get(function_name, context);
|
||||
auto function = FunctionFactory::instance().get(function_name, getContext());
|
||||
function_node.resolveAsFunction(function->build(function_node.getArgumentColumns()));
|
||||
}
|
||||
|
||||
ContextPtr & context;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Transform functions to subcolumns.
|
||||
/** Transform functions to subcolumns. Enabled using setting optimize_functions_to_subcolumns.
|
||||
* It can help to reduce amount of read data.
|
||||
*
|
||||
* Example: SELECT tupleElement(column, subcolumn) FROM test_table;
|
||||
|
||||
@ -26,16 +26,22 @@ namespace ErrorCodes
|
||||
namespace
|
||||
{
|
||||
|
||||
class FuseFunctionsVisitor : public InDepthQueryTreeVisitor<FuseFunctionsVisitor>
|
||||
class FuseFunctionsVisitor : public InDepthQueryTreeVisitorWithContext<FuseFunctionsVisitor>
|
||||
{
|
||||
public:
|
||||
using Base = InDepthQueryTreeVisitorWithContext<FuseFunctionsVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
explicit FuseFunctionsVisitor(const std::unordered_set<String> names_to_collect_)
|
||||
: names_to_collect(names_to_collect_)
|
||||
explicit FuseFunctionsVisitor(const std::unordered_set<String> names_to_collect_, ContextPtr context)
|
||||
: Base(std::move(context))
|
||||
, names_to_collect(names_to_collect_)
|
||||
{}
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_syntax_fuse_functions)
|
||||
return;
|
||||
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
if (!function_node || !function_node->isAggregateFunction() || !names_to_collect.contains(function_node->getFunctionName()))
|
||||
return;
|
||||
@ -201,7 +207,7 @@ FunctionNodePtr createFusedQuantilesNode(std::vector<QueryTreeNodePtr *> & nodes
|
||||
|
||||
void tryFuseSumCountAvg(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
FuseFunctionsVisitor visitor({"sum", "count", "avg"});
|
||||
FuseFunctionsVisitor visitor({"sum", "count", "avg"}, context);
|
||||
visitor.visit(query_tree_node);
|
||||
|
||||
for (auto & [argument, nodes] : visitor.argument_to_functions_mapping)
|
||||
@ -220,7 +226,7 @@ void tryFuseSumCountAvg(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
|
||||
void tryFuseQuantiles(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
FuseFunctionsVisitor visitor_quantile({"quantile"});
|
||||
FuseFunctionsVisitor visitor_quantile({"quantile"}, context);
|
||||
visitor_quantile.visit(query_tree_node);
|
||||
|
||||
for (auto & [argument, nodes_set] : visitor_quantile.argument_to_functions_mapping)
|
||||
|
||||
@ -12,15 +12,22 @@ namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
class IfChainToMultiIfPassVisitor : public InDepthQueryTreeVisitor<IfChainToMultiIfPassVisitor>
|
||||
class IfChainToMultiIfPassVisitor : public InDepthQueryTreeVisitorWithContext<IfChainToMultiIfPassVisitor>
|
||||
{
|
||||
public:
|
||||
explicit IfChainToMultiIfPassVisitor(FunctionOverloadResolverPtr multi_if_function_ptr_)
|
||||
: multi_if_function_ptr(std::move(multi_if_function_ptr_))
|
||||
using Base = InDepthQueryTreeVisitorWithContext<IfChainToMultiIfPassVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
explicit IfChainToMultiIfPassVisitor(FunctionOverloadResolverPtr multi_if_function_ptr_, ContextPtr context)
|
||||
: Base(std::move(context))
|
||||
, multi_if_function_ptr(std::move(multi_if_function_ptr_))
|
||||
{}
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_if_chain_to_multiif)
|
||||
return;
|
||||
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
if (!function_node || function_node->getFunctionName() != "if" || function_node->getArguments().getNodes().size() != 3)
|
||||
return;
|
||||
@ -68,7 +75,8 @@ private:
|
||||
|
||||
void IfChainToMultiIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
IfChainToMultiIfPassVisitor visitor(FunctionFactory::instance().get("multiIf", context));
|
||||
auto multi_if_function_ptr = FunctionFactory::instance().get("multiIf", context);
|
||||
IfChainToMultiIfPassVisitor visitor(std::move(multi_if_function_ptr), std::move(context));
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
|
||||
@ -107,21 +107,24 @@ void wrapIntoToString(FunctionNode & function_node, QueryTreeNodePtr arg, Contex
|
||||
assert(isString(function_node.getResultType()));
|
||||
}
|
||||
|
||||
class ConvertStringsToEnumVisitor : public InDepthQueryTreeVisitor<ConvertStringsToEnumVisitor>
|
||||
class ConvertStringsToEnumVisitor : public InDepthQueryTreeVisitorWithContext<ConvertStringsToEnumVisitor>
|
||||
{
|
||||
public:
|
||||
explicit ConvertStringsToEnumVisitor(ContextPtr context_)
|
||||
: context(std::move(context_))
|
||||
{
|
||||
}
|
||||
using Base = InDepthQueryTreeVisitorWithContext<ConvertStringsToEnumVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_if_transform_strings_to_enum)
|
||||
return;
|
||||
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
|
||||
if (!function_node)
|
||||
return;
|
||||
|
||||
const auto & context = getContext();
|
||||
|
||||
/// to preserve return type (String) of the current function_node, we wrap the newly
|
||||
/// generated function nodes into toString
|
||||
|
||||
@ -198,16 +201,13 @@ public:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
ContextPtr context;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void IfTransformStringsToEnumPass::run(QueryTreeNodePtr query, ContextPtr context)
|
||||
{
|
||||
ConvertStringsToEnumVisitor visitor(context);
|
||||
ConvertStringsToEnumVisitor visitor(std::move(context));
|
||||
visitor.visit(query);
|
||||
}
|
||||
|
||||
|
||||
@ -10,15 +10,22 @@ namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
class MultiIfToIfVisitor : public InDepthQueryTreeVisitor<MultiIfToIfVisitor>
|
||||
class MultiIfToIfVisitor : public InDepthQueryTreeVisitorWithContext<MultiIfToIfVisitor>
|
||||
{
|
||||
public:
|
||||
explicit MultiIfToIfVisitor(FunctionOverloadResolverPtr if_function_ptr_)
|
||||
: if_function_ptr(if_function_ptr_)
|
||||
using Base = InDepthQueryTreeVisitorWithContext<MultiIfToIfVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
explicit MultiIfToIfVisitor(FunctionOverloadResolverPtr if_function_ptr_, ContextPtr context)
|
||||
: Base(std::move(context))
|
||||
, if_function_ptr(std::move(if_function_ptr_))
|
||||
{}
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_multiif_to_if)
|
||||
return;
|
||||
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
if (!function_node || function_node->getFunctionName() != "multiIf")
|
||||
return;
|
||||
@ -38,7 +45,8 @@ private:
|
||||
|
||||
void MultiIfToIfPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
MultiIfToIfVisitor visitor(FunctionFactory::instance().get("if", context));
|
||||
auto if_function_ptr = FunctionFactory::instance().get("if", context);
|
||||
MultiIfToIfVisitor visitor(std::move(if_function_ptr), std::move(context));
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
|
||||
@ -14,12 +14,17 @@ namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
class NormalizeCountVariantsVisitor : public InDepthQueryTreeVisitor<NormalizeCountVariantsVisitor>
|
||||
class NormalizeCountVariantsVisitor : public InDepthQueryTreeVisitorWithContext<NormalizeCountVariantsVisitor>
|
||||
{
|
||||
public:
|
||||
explicit NormalizeCountVariantsVisitor(ContextPtr context_) : context(std::move(context_)) {}
|
||||
using Base = InDepthQueryTreeVisitorWithContext<NormalizeCountVariantsVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_normalize_count_variants)
|
||||
return;
|
||||
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
if (!function_node || !function_node->isAggregateFunction() || (function_node->getFunctionName() != "count" && function_node->getFunctionName() != "sum"))
|
||||
return;
|
||||
@ -42,15 +47,13 @@ public:
|
||||
else if (function_node->getFunctionName() == "sum" &&
|
||||
first_argument_constant_literal.getType() == Field::Types::UInt64 &&
|
||||
first_argument_constant_literal.get<UInt64>() == 1 &&
|
||||
!context->getSettingsRef().aggregate_functions_null_for_empty)
|
||||
!getSettings().aggregate_functions_null_for_empty)
|
||||
{
|
||||
resolveAsCountAggregateFunction(*function_node);
|
||||
function_node->getArguments().getNodes().clear();
|
||||
}
|
||||
}
|
||||
private:
|
||||
ContextPtr context;
|
||||
|
||||
static inline void resolveAsCountAggregateFunction(FunctionNode & function_node)
|
||||
{
|
||||
AggregateFunctionProperties properties;
|
||||
|
||||
@ -1,26 +1,33 @@
|
||||
#include <Analyzer/Passes/OptimizeGroupByFunctionKeysPass.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <queue>
|
||||
|
||||
#include <Analyzer/FunctionNode.h>
|
||||
#include <Analyzer/HashUtils.h>
|
||||
#include <Analyzer/IQueryTreeNode.h>
|
||||
#include <Analyzer/InDepthQueryTreeVisitor.h>
|
||||
#include <Analyzer/QueryNode.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <queue>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class OptimizeGroupByFunctionKeysVisitor : public InDepthQueryTreeVisitor<OptimizeGroupByFunctionKeysVisitor>
|
||||
class OptimizeGroupByFunctionKeysVisitor : public InDepthQueryTreeVisitorWithContext<OptimizeGroupByFunctionKeysVisitor>
|
||||
{
|
||||
public:
|
||||
using Base = InDepthQueryTreeVisitorWithContext<OptimizeGroupByFunctionKeysVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
static bool needChildVisit(QueryTreeNodePtr & /*parent*/, QueryTreeNodePtr & child)
|
||||
{
|
||||
return !child->as<FunctionNode>();
|
||||
}
|
||||
|
||||
static void visitImpl(QueryTreeNodePtr & node)
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_group_by_function_keys)
|
||||
return;
|
||||
|
||||
auto * query = node->as<QueryNode>();
|
||||
if (!query)
|
||||
return;
|
||||
@ -41,11 +48,10 @@ public:
|
||||
optimizeGroupingSet(group_by);
|
||||
}
|
||||
private:
|
||||
|
||||
struct NodeWithInfo
|
||||
{
|
||||
QueryTreeNodePtr node;
|
||||
bool parents_are_only_deterministic;
|
||||
bool parents_are_only_deterministic = false;
|
||||
};
|
||||
|
||||
static bool canBeEliminated(QueryTreeNodePtr & node, const QueryTreeNodePtrWithHashSet & group_by_keys)
|
||||
@ -64,7 +70,7 @@ private:
|
||||
// TODO: Also process CONSTANT here. We can simplify GROUP BY x, x + 1 to GROUP BY x.
|
||||
while (!candidates.empty())
|
||||
{
|
||||
auto [candidate, deterministic_context] = candidates.back();
|
||||
auto [candidate, parents_are_only_deterministic] = candidates.back();
|
||||
candidates.pop_back();
|
||||
|
||||
bool found = group_by_keys.contains(candidate);
|
||||
@ -80,7 +86,7 @@ private:
|
||||
|
||||
if (!found)
|
||||
{
|
||||
bool is_deterministic_function = deterministic_context && function->getFunction()->isDeterministicInScopeOfQuery();
|
||||
bool is_deterministic_function = parents_are_only_deterministic && function->getFunction()->isDeterministicInScopeOfQuery();
|
||||
for (auto it = arguments.rbegin(); it != arguments.rend(); ++it)
|
||||
candidates.push_back({ *it, is_deterministic_function });
|
||||
}
|
||||
@ -91,7 +97,7 @@ private:
|
||||
return false;
|
||||
break;
|
||||
case QueryTreeNodeType::CONSTANT:
|
||||
if (!deterministic_context)
|
||||
if (!parents_are_only_deterministic)
|
||||
return false;
|
||||
break;
|
||||
default:
|
||||
@ -117,9 +123,10 @@ private:
|
||||
}
|
||||
};
|
||||
|
||||
void OptimizeGroupByFunctionKeysPass::run(QueryTreeNodePtr query_tree_node, ContextPtr /*context*/)
|
||||
void OptimizeGroupByFunctionKeysPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
OptimizeGroupByFunctionKeysVisitor().visit(query_tree_node);
|
||||
OptimizeGroupByFunctionKeysVisitor visitor(std::move(context));
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
#include <Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h>
|
||||
|
||||
#include <Functions/IFunction.h>
|
||||
|
||||
#include <Analyzer/ColumnNode.h>
|
||||
#include <Analyzer/FunctionNode.h>
|
||||
#include <Analyzer/HashUtils.h>
|
||||
#include <Analyzer/InDepthQueryTreeVisitor.h>
|
||||
#include <Analyzer/QueryNode.h>
|
||||
#include <Analyzer/SortNode.h>
|
||||
#include <Functions/IFunction.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -13,9 +15,12 @@ namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
class OptimizeRedundantFunctionsInOrderByVisitor : public InDepthQueryTreeVisitor<OptimizeRedundantFunctionsInOrderByVisitor>
|
||||
class OptimizeRedundantFunctionsInOrderByVisitor : public InDepthQueryTreeVisitorWithContext<OptimizeRedundantFunctionsInOrderByVisitor>
|
||||
{
|
||||
public:
|
||||
using Base = InDepthQueryTreeVisitorWithContext<OptimizeRedundantFunctionsInOrderByVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
static bool needChildVisit(QueryTreeNodePtr & node, QueryTreeNodePtr & /*parent*/)
|
||||
{
|
||||
if (node->as<FunctionNode>())
|
||||
@ -25,6 +30,9 @@ public:
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_redundant_functions_in_order_by)
|
||||
return;
|
||||
|
||||
auto * query = node->as<QueryNode>();
|
||||
if (!query)
|
||||
return;
|
||||
@ -116,9 +124,10 @@ private:
|
||||
|
||||
}
|
||||
|
||||
void OptimizeRedundantFunctionsInOrderByPass::run(QueryTreeNodePtr query_tree_node, ContextPtr /*context*/)
|
||||
void OptimizeRedundantFunctionsInOrderByPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
OptimizeRedundantFunctionsInOrderByVisitor().visit(query_tree_node);
|
||||
OptimizeRedundantFunctionsInOrderByVisitor visitor(std::move(context));
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1943,7 +1943,7 @@ void QueryAnalyzer::validateTableExpressionModifiers(const QueryTreeNodePtr & ta
|
||||
|
||||
if (!table_node && !table_function_node && !query_node && !union_node)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Unexpected table expression. Expected table, table function, query or union node. Actual {}",
|
||||
"Unexpected table expression. Expected table, table function, query or union node. Table node: {}, scope node: {}",
|
||||
table_expression_node->formatASTForErrorMessage(),
|
||||
scope.scope_node->formatASTForErrorMessage());
|
||||
|
||||
@ -4366,12 +4366,9 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
|
||||
{
|
||||
if (!AggregateFunctionFactory::instance().isAggregateFunctionName(function_name))
|
||||
{
|
||||
std::string error_message = fmt::format("Aggregate function with name '{}' does not exists. In scope {}",
|
||||
function_name,
|
||||
scope.scope_node->formatASTForErrorMessage());
|
||||
|
||||
AggregateFunctionFactory::instance().appendHintsMessage(error_message, function_name);
|
||||
throw Exception(ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION, error_message);
|
||||
throw Exception(ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION, "Aggregate function with name '{}' does not exists. In scope {}{}",
|
||||
function_name, scope.scope_node->formatASTForErrorMessage(),
|
||||
getHintsErrorMessageSuffix(AggregateFunctionFactory::instance().getHints(function_name)));
|
||||
}
|
||||
|
||||
if (!function_lambda_arguments_indexes.empty())
|
||||
@ -5726,7 +5723,7 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node,
|
||||
case QueryTreeNodeType::IDENTIFIER:
|
||||
{
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Identifiers in FROM section must be already resolved. In scope {}",
|
||||
"Identifiers in FROM section must be already resolved. Node {}, scope {}",
|
||||
join_tree_node->formatASTForErrorMessage(),
|
||||
scope.scope_node->formatASTForErrorMessage());
|
||||
}
|
||||
|
||||
@ -20,15 +20,17 @@ namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
class SumIfToCountIfVisitor : public InDepthQueryTreeVisitor<SumIfToCountIfVisitor>
|
||||
class SumIfToCountIfVisitor : public InDepthQueryTreeVisitorWithContext<SumIfToCountIfVisitor>
|
||||
{
|
||||
public:
|
||||
explicit SumIfToCountIfVisitor(ContextPtr & context_)
|
||||
: context(context_)
|
||||
{}
|
||||
using Base = InDepthQueryTreeVisitorWithContext<SumIfToCountIfVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_rewrite_sum_if_to_count_if)
|
||||
return;
|
||||
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
if (!function_node || !function_node->isAggregateFunction())
|
||||
return;
|
||||
@ -56,7 +58,7 @@ public:
|
||||
if (!isInt64OrUInt64FieldType(constant_value_literal.getType()))
|
||||
return;
|
||||
|
||||
if (constant_value_literal.get<UInt64>() != 1 || context->getSettingsRef().aggregate_functions_null_for_empty)
|
||||
if (constant_value_literal.get<UInt64>() != 1 || getSettings().aggregate_functions_null_for_empty)
|
||||
return;
|
||||
|
||||
function_node_arguments_nodes[0] = std::move(function_node_arguments_nodes[1]);
|
||||
@ -122,7 +124,7 @@ public:
|
||||
auto & not_function_arguments = not_function->getArguments().getNodes();
|
||||
not_function_arguments.push_back(nested_if_function_arguments_nodes[0]);
|
||||
|
||||
not_function->resolveAsFunction(FunctionFactory::instance().get("not", context)->build(not_function->getArgumentColumns()));
|
||||
not_function->resolveAsFunction(FunctionFactory::instance().get("not", getContext())->build(not_function->getArgumentColumns()));
|
||||
|
||||
function_node_arguments_nodes[0] = std::move(not_function);
|
||||
function_node_arguments_nodes.resize(1);
|
||||
@ -143,8 +145,6 @@ private:
|
||||
|
||||
function_node.resolveAsAggregateFunction(std::move(aggregate_function));
|
||||
}
|
||||
|
||||
ContextPtr & context;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -25,11 +25,17 @@ bool isUniqFunction(const String & function_name)
|
||||
function_name == "uniqTheta";
|
||||
}
|
||||
|
||||
class UniqInjectiveFunctionsEliminationVisitor : public InDepthQueryTreeVisitor<UniqInjectiveFunctionsEliminationVisitor>
|
||||
class UniqInjectiveFunctionsEliminationVisitor : public InDepthQueryTreeVisitorWithContext<UniqInjectiveFunctionsEliminationVisitor>
|
||||
{
|
||||
public:
|
||||
static void visitImpl(QueryTreeNodePtr & node)
|
||||
using Base = InDepthQueryTreeVisitorWithContext<UniqInjectiveFunctionsEliminationVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (!getSettings().optimize_injective_functions_inside_uniq)
|
||||
return;
|
||||
|
||||
auto * function_node = node->as<FunctionNode>();
|
||||
if (!function_node || !function_node->isAggregateFunction() || !isUniqFunction(function_node->getFunctionName()))
|
||||
return;
|
||||
@ -81,9 +87,9 @@ public:
|
||||
|
||||
}
|
||||
|
||||
void UniqInjectiveFunctionsEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr)
|
||||
void UniqInjectiveFunctionsEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
|
||||
{
|
||||
UniqInjectiveFunctionsEliminationVisitor visitor;
|
||||
UniqInjectiveFunctionsEliminationVisitor visitor(std::move(context));
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#include <Analyzer/QueryNode.h>
|
||||
|
||||
#include <fmt/core.h>
|
||||
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/FieldVisitorToString.h>
|
||||
|
||||
@ -17,7 +19,6 @@
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
|
||||
#include <Analyzer/Utils.h>
|
||||
#include <fmt/core.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -36,7 +37,7 @@ QueryNode::QueryNode(ContextMutablePtr context_, SettingsChanges settings_change
|
||||
}
|
||||
|
||||
QueryNode::QueryNode(ContextMutablePtr context_)
|
||||
: QueryNode(context_, {} /*settings_changes*/)
|
||||
: QueryNode(std::move(context_), {} /*settings_changes*/)
|
||||
{}
|
||||
|
||||
void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const
|
||||
@ -185,10 +186,7 @@ void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s
|
||||
{
|
||||
buffer << '\n' << std::string(indent + 2, ' ') << "SETTINGS";
|
||||
for (const auto & change : settings_changes)
|
||||
{
|
||||
buffer << fmt::format(" {}={}", change.name, toString(change.value));
|
||||
}
|
||||
buffer << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include <memory>
|
||||
#include <Analyzer/QueryTreePassManager.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
|
||||
#include <IO/WriteHelpers.h>
|
||||
@ -133,7 +134,6 @@ private:
|
||||
* TODO: Support setting optimize_aggregators_of_group_by_keys.
|
||||
* TODO: Support setting optimize_duplicate_order_by_and_distinct.
|
||||
* TODO: Support setting optimize_monotonous_functions_in_order_by.
|
||||
* TODO: Support settings.optimize_or_like_chain.
|
||||
* TODO: Add optimizations based on function semantics. Example: SELECT * FROM test_table WHERE id != id. (id is not nullable column).
|
||||
*/
|
||||
|
||||
@ -210,53 +210,31 @@ void QueryTreePassManager::dump(WriteBuffer & buffer, size_t up_to_pass_index)
|
||||
|
||||
void addQueryTreePasses(QueryTreePassManager & manager)
|
||||
{
|
||||
auto context = manager.getContext();
|
||||
const auto & settings = context->getSettingsRef();
|
||||
|
||||
manager.addPass(std::make_unique<QueryAnalysisPass>());
|
||||
manager.addPass(std::make_unique<FunctionToSubcolumnsPass>());
|
||||
|
||||
if (settings.optimize_functions_to_subcolumns)
|
||||
manager.addPass(std::make_unique<FunctionToSubcolumnsPass>());
|
||||
|
||||
if (settings.count_distinct_optimization)
|
||||
manager.addPass(std::make_unique<CountDistinctPass>());
|
||||
|
||||
if (settings.optimize_rewrite_sum_if_to_count_if)
|
||||
manager.addPass(std::make_unique<SumIfToCountIfPass>());
|
||||
|
||||
if (settings.optimize_normalize_count_variants)
|
||||
manager.addPass(std::make_unique<NormalizeCountVariantsPass>());
|
||||
manager.addPass(std::make_unique<CountDistinctPass>());
|
||||
manager.addPass(std::make_unique<SumIfToCountIfPass>());
|
||||
manager.addPass(std::make_unique<NormalizeCountVariantsPass>());
|
||||
|
||||
manager.addPass(std::make_unique<CustomizeFunctionsPass>());
|
||||
|
||||
if (settings.optimize_arithmetic_operations_in_aggregate_functions)
|
||||
manager.addPass(std::make_unique<AggregateFunctionsArithmericOperationsPass>());
|
||||
|
||||
if (settings.optimize_injective_functions_inside_uniq)
|
||||
manager.addPass(std::make_unique<UniqInjectiveFunctionsEliminationPass>());
|
||||
|
||||
if (settings.optimize_group_by_function_keys)
|
||||
manager.addPass(std::make_unique<OptimizeGroupByFunctionKeysPass>());
|
||||
|
||||
if (settings.optimize_multiif_to_if)
|
||||
manager.addPass(std::make_unique<MultiIfToIfPass>());
|
||||
manager.addPass(std::make_unique<AggregateFunctionsArithmericOperationsPass>());
|
||||
manager.addPass(std::make_unique<UniqInjectiveFunctionsEliminationPass>());
|
||||
manager.addPass(std::make_unique<OptimizeGroupByFunctionKeysPass>());
|
||||
|
||||
manager.addPass(std::make_unique<MultiIfToIfPass>());
|
||||
manager.addPass(std::make_unique<IfConstantConditionPass>());
|
||||
manager.addPass(std::make_unique<IfChainToMultiIfPass>());
|
||||
|
||||
if (settings.optimize_if_chain_to_multiif)
|
||||
manager.addPass(std::make_unique<IfChainToMultiIfPass>());
|
||||
|
||||
if (settings.optimize_redundant_functions_in_order_by)
|
||||
manager.addPass(std::make_unique<OptimizeRedundantFunctionsInOrderByPass>());
|
||||
manager.addPass(std::make_unique<OptimizeRedundantFunctionsInOrderByPass>());
|
||||
|
||||
manager.addPass(std::make_unique<OrderByTupleEliminationPass>());
|
||||
manager.addPass(std::make_unique<OrderByLimitByDuplicateEliminationPass>());
|
||||
|
||||
if (settings.optimize_syntax_fuse_functions)
|
||||
manager.addPass(std::make_unique<FuseFunctionsPass>());
|
||||
manager.addPass(std::make_unique<FuseFunctionsPass>());
|
||||
|
||||
if (settings.optimize_if_transform_strings_to_enum)
|
||||
manager.addPass(std::make_unique<IfTransformStringsToEnumPass>());
|
||||
manager.addPass(std::make_unique<IfTransformStringsToEnumPass>());
|
||||
|
||||
manager.addPass(std::make_unique<ConvertOrLikeChainPass>());
|
||||
|
||||
|
||||
@ -79,7 +79,7 @@ namespace
|
||||
request.SetMaxKeys(1);
|
||||
auto outcome = client.ListObjects(request);
|
||||
if (!outcome.IsSuccess())
|
||||
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
throw Exception::createDeprecated(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
return outcome.GetResult().GetContents();
|
||||
}
|
||||
|
||||
@ -233,7 +233,7 @@ void BackupWriterS3::removeFile(const String & file_name)
|
||||
request.SetKey(fs::path(s3_uri.key) / file_name);
|
||||
auto outcome = client->DeleteObject(request);
|
||||
if (!outcome.IsSuccess() && !isNotFoundError(outcome.GetError().GetErrorType()))
|
||||
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
throw Exception::createDeprecated(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
}
|
||||
|
||||
void BackupWriterS3::removeFiles(const Strings & file_names)
|
||||
@ -291,7 +291,7 @@ void BackupWriterS3::removeFilesBatch(const Strings & file_names)
|
||||
|
||||
auto outcome = client->DeleteObjects(request);
|
||||
if (!outcome.IsSuccess() && !isNotFoundError(outcome.GetError().GetErrorType()))
|
||||
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
throw Exception::createDeprecated(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -93,7 +93,7 @@ namespace
|
||||
catch (...)
|
||||
{
|
||||
if (coordination)
|
||||
coordination->setError(current_host, Exception{getCurrentExceptionCode(), getCurrentExceptionMessage(true, true)});
|
||||
coordination->setError(current_host, Exception(getCurrentExceptionMessageAndPattern(true, true), getCurrentExceptionCode()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -451,7 +451,7 @@ void ClientBase::onData(Block & block, ASTPtr parsed_query)
|
||||
catch (const Exception &)
|
||||
{
|
||||
/// Catch client errors like NO_ROW_DELIMITER
|
||||
throw LocalFormatError(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode());
|
||||
throw LocalFormatError(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode());
|
||||
}
|
||||
|
||||
/// Received data block is immediately displayed to the user.
|
||||
@ -629,7 +629,7 @@ try
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
throw LocalFormatError(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode());
|
||||
throw LocalFormatError(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode());
|
||||
}
|
||||
|
||||
|
||||
@ -1897,7 +1897,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
|
||||
{
|
||||
// Surprisingly, this is a client error. A server error would
|
||||
// have been reported without throwing (see onReceiveSeverException()).
|
||||
client_exception = std::make_unique<Exception>(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode());
|
||||
client_exception = std::make_unique<Exception>(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode());
|
||||
have_error = true;
|
||||
}
|
||||
|
||||
|
||||
@ -187,7 +187,7 @@ void LocalConnection::sendQuery(
|
||||
catch (...)
|
||||
{
|
||||
state->io.onException();
|
||||
state->exception = std::make_unique<Exception>("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
|
||||
state->exception = std::make_unique<Exception>(ErrorCodes::UNKNOWN_EXCEPTION, "Unknown exception");
|
||||
}
|
||||
}
|
||||
|
||||
@ -291,7 +291,7 @@ bool LocalConnection::poll(size_t)
|
||||
catch (...)
|
||||
{
|
||||
state->io.onException();
|
||||
state->exception = std::make_unique<Exception>("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
|
||||
state->exception = std::make_unique<Exception>(ErrorCodes::UNKNOWN_EXCEPTION, "Unknown exception");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -83,7 +83,7 @@ template <is_decimal T>
|
||||
UInt64 ColumnDecimal<T>::get64([[maybe_unused]] size_t n) const
|
||||
{
|
||||
if constexpr (sizeof(T) > sizeof(UInt64))
|
||||
throw Exception(String("Method get64 is not supported for ") + getFamilyName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method get64 is not supported for {}", getFamilyName());
|
||||
else
|
||||
return static_cast<NativeT>(data[n]);
|
||||
}
|
||||
|
||||
@ -35,7 +35,7 @@ ColumnNullable::ColumnNullable(MutableColumnPtr && nested_column_, MutableColumn
|
||||
nested_column = getNestedColumn().convertToFullColumnIfConst();
|
||||
|
||||
if (!getNestedColumn().canBeInsideNullable())
|
||||
throw Exception{getNestedColumn().getName() + " cannot be inside Nullable column", ErrorCodes::ILLEGAL_COLUMN};
|
||||
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "{} cannot be inside Nullable column", getNestedColumn().getName());
|
||||
|
||||
if (isColumnConst(*null_map))
|
||||
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "ColumnNullable cannot have constant null map");
|
||||
|
||||
@ -331,46 +331,14 @@ size_t ColumnUnique<ColumnType>::getNullValueIndex() const
|
||||
template <typename ColumnType>
|
||||
size_t ColumnUnique<ColumnType>::uniqueInsert(const Field & x)
|
||||
{
|
||||
class FieldVisitorGetData : public StaticVisitor<>
|
||||
{
|
||||
public:
|
||||
StringRef res;
|
||||
|
||||
[[noreturn]] static void throwUnsupported()
|
||||
{
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unsupported field type");
|
||||
}
|
||||
|
||||
[[noreturn]] void operator() (const Null &) { throwUnsupported(); }
|
||||
[[noreturn]] void operator() (const Array &) { throwUnsupported(); }
|
||||
[[noreturn]] void operator() (const Tuple &) { throwUnsupported(); }
|
||||
[[noreturn]] void operator() (const Map &) { throwUnsupported(); }
|
||||
[[noreturn]] void operator() (const Object &) { throwUnsupported(); }
|
||||
[[noreturn]] void operator() (const AggregateFunctionStateData &) { throwUnsupported(); }
|
||||
void operator() (const String & x) { res = {x.data(), x.size()}; }
|
||||
void operator() (const UInt64 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const UInt128 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const UInt256 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const Int64 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const Int128 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const Int256 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const UUID & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const IPv4 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const IPv6 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const Float64 & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const DecimalField<Decimal32> & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const DecimalField<Decimal64> & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const DecimalField<Decimal128> & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const DecimalField<Decimal256> & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
void operator() (const bool & x) { res = {reinterpret_cast<const char *>(&x), sizeof(x)}; }
|
||||
};
|
||||
|
||||
if (x.isNull())
|
||||
return getNullValueIndex();
|
||||
|
||||
FieldVisitorGetData visitor;
|
||||
applyVisitor(visitor, x);
|
||||
return uniqueInsertData(visitor.res.data, visitor.res.size);
|
||||
auto single_value_column = column_holder->cloneEmpty();
|
||||
single_value_column->insert(x);
|
||||
auto single_value_data = single_value_column->getDataAt(0);
|
||||
|
||||
return uniqueInsertData(single_value_data.data, single_value_data.size);
|
||||
}
|
||||
|
||||
template <typename ColumnType>
|
||||
|
||||
50
src/Columns/tests/gtest_low_cardinality.cpp
Normal file
50
src/Columns/tests/gtest_low_cardinality.cpp
Normal file
@ -0,0 +1,50 @@
|
||||
#include <Columns/ColumnLowCardinality.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
using namespace DB;
|
||||
|
||||
template <typename T>
|
||||
void testLowCardinalityNumberInsert(const DataTypePtr & data_type)
|
||||
{
|
||||
auto low_cardinality_type = std::make_shared<DataTypeLowCardinality>(data_type);
|
||||
auto column = low_cardinality_type->createColumn();
|
||||
|
||||
column->insert(static_cast<T>(15));
|
||||
column->insert(static_cast<T>(20));
|
||||
column->insert(static_cast<T>(25));
|
||||
|
||||
Field value;
|
||||
column->get(0, value);
|
||||
ASSERT_EQ(value.get<T>(), 15);
|
||||
|
||||
column->get(1, value);
|
||||
ASSERT_EQ(value.get<T>(), 20);
|
||||
|
||||
column->get(2, value);
|
||||
ASSERT_EQ(value.get<T>(), 25);
|
||||
}
|
||||
|
||||
TEST(ColumnLowCardinality, Insert)
|
||||
{
|
||||
testLowCardinalityNumberInsert<UInt8>(std::make_shared<DataTypeUInt8>());
|
||||
testLowCardinalityNumberInsert<UInt16>(std::make_shared<DataTypeUInt16>());
|
||||
testLowCardinalityNumberInsert<UInt32>(std::make_shared<DataTypeUInt32>());
|
||||
testLowCardinalityNumberInsert<UInt64>(std::make_shared<DataTypeUInt64>());
|
||||
testLowCardinalityNumberInsert<UInt128>(std::make_shared<DataTypeUInt128>());
|
||||
testLowCardinalityNumberInsert<UInt256>(std::make_shared<DataTypeUInt256>());
|
||||
|
||||
testLowCardinalityNumberInsert<Int8>(std::make_shared<DataTypeInt8>());
|
||||
testLowCardinalityNumberInsert<Int16>(std::make_shared<DataTypeInt16>());
|
||||
testLowCardinalityNumberInsert<Int32>(std::make_shared<DataTypeInt32>());
|
||||
testLowCardinalityNumberInsert<Int64>(std::make_shared<DataTypeInt64>());
|
||||
testLowCardinalityNumberInsert<Int128>(std::make_shared<DataTypeInt128>());
|
||||
testLowCardinalityNumberInsert<Int256>(std::make_shared<DataTypeInt256>());
|
||||
|
||||
testLowCardinalityNumberInsert<Float32>(std::make_shared<DataTypeFloat32>());
|
||||
testLowCardinalityNumberInsert<Float64>(std::make_shared<DataTypeFloat64>());
|
||||
}
|
||||
@ -207,8 +207,9 @@ private:
|
||||
if (size >= MMAP_THRESHOLD)
|
||||
{
|
||||
if (alignment > mmap_min_alignment)
|
||||
throw DB::Exception(fmt::format("Too large alignment {}: more than page size when allocating {}.",
|
||||
ReadableSize(alignment), ReadableSize(size)), DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS,
|
||||
"Too large alignment {}: more than page size when allocating {}.",
|
||||
ReadableSize(alignment), ReadableSize(size));
|
||||
|
||||
buf = mmap(getMmapHint(), size, PROT_READ | PROT_WRITE,
|
||||
mmap_flags, -1, 0);
|
||||
|
||||
@ -140,7 +140,7 @@ void CancelToken::raise()
|
||||
{
|
||||
std::unique_lock lock(signal_mutex);
|
||||
if (exception_code != 0)
|
||||
throw DB::Exception(
|
||||
throw DB::Exception::createRuntime(
|
||||
std::exchange(exception_code, 0),
|
||||
std::exchange(exception_message, {}));
|
||||
else
|
||||
|
||||
@ -88,7 +88,7 @@ public:
|
||||
{
|
||||
/// A more understandable error message.
|
||||
if (e.code() == DB::ErrorCodes::CANNOT_READ_ALL_DATA || e.code() == DB::ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF)
|
||||
throw DB::ParsingException("File " + path + " is empty. You must fill it manually with appropriate value.", e.code());
|
||||
throw DB::ParsingException(e.code(), "File {} is empty. You must fill it manually with appropriate value.", path);
|
||||
else
|
||||
throw;
|
||||
}
|
||||
|
||||
@ -252,7 +252,7 @@ uint64_t readOffset(std::string_view & sp, bool is64_bit)
|
||||
// Read "len" bytes
|
||||
std::string_view readBytes(std::string_view & sp, uint64_t len)
|
||||
{
|
||||
SAFE_CHECK(len <= sp.size(), "invalid string length: " + std::to_string(len) + " vs. " + std::to_string(sp.size()));
|
||||
SAFE_CHECK(len <= sp.size(), "invalid string length: {} vs. {}", len, sp.size());
|
||||
std::string_view ret(sp.data(), len);
|
||||
sp.remove_prefix(len);
|
||||
return ret;
|
||||
@ -953,7 +953,7 @@ bool Dwarf::findDebugInfoOffset(uintptr_t address, std::string_view aranges, uin
|
||||
|
||||
Dwarf::Die Dwarf::getDieAtOffset(const CompilationUnit & cu, uint64_t offset) const
|
||||
{
|
||||
SAFE_CHECK(offset < info_.size(), fmt::format("unexpected offset {}, info size {}", offset, info_.size()));
|
||||
SAFE_CHECK(offset < info_.size(), "unexpected offset {}, info size {}", offset, info_.size());
|
||||
Die die;
|
||||
std::string_view sp{info_.data() + offset, cu.offset + cu.size - offset};
|
||||
die.offset = offset;
|
||||
|
||||
@ -188,7 +188,7 @@ static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string
|
||||
{
|
||||
PreformattedMessage message = getCurrentExceptionMessageAndPattern(true);
|
||||
if (!start_of_message.empty())
|
||||
message.message = fmt::format("{}: {}", start_of_message, message.message);
|
||||
message.text = fmt::format("{}: {}", start_of_message, message.text);
|
||||
|
||||
LOG_ERROR(logger, message);
|
||||
}
|
||||
@ -339,7 +339,7 @@ std::string getExtraExceptionInfo(const std::exception & e)
|
||||
|
||||
std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded_stacktrace /*= false*/, bool with_extra_info /*= true*/)
|
||||
{
|
||||
return getCurrentExceptionMessageAndPattern(with_stacktrace, check_embedded_stacktrace, with_extra_info).message;
|
||||
return getCurrentExceptionMessageAndPattern(with_stacktrace, check_embedded_stacktrace, with_extra_info).text;
|
||||
}
|
||||
|
||||
PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, bool check_embedded_stacktrace /*= false*/, bool with_extra_info /*= true*/)
|
||||
@ -481,7 +481,7 @@ void tryLogException(std::exception_ptr e, Poco::Logger * logger, const std::str
|
||||
|
||||
std::string getExceptionMessage(const Exception & e, bool with_stacktrace, bool check_embedded_stacktrace)
|
||||
{
|
||||
return getExceptionMessageAndPattern(e, with_stacktrace, check_embedded_stacktrace).message;
|
||||
return getExceptionMessageAndPattern(e, with_stacktrace, check_embedded_stacktrace).text;
|
||||
}
|
||||
|
||||
PreformattedMessage getExceptionMessageAndPattern(const Exception & e, bool with_stacktrace, bool check_embedded_stacktrace)
|
||||
@ -577,10 +577,6 @@ ParsingException::ParsingException(const std::string & msg, int code)
|
||||
: Exception(msg, code)
|
||||
{
|
||||
}
|
||||
ParsingException::ParsingException(int code, const std::string & message)
|
||||
: Exception(message, code)
|
||||
{
|
||||
}
|
||||
|
||||
/// We use additional field formatted_message_ to make this method const.
|
||||
std::string ParsingException::displayText() const
|
||||
|
||||
@ -16,26 +16,6 @@
|
||||
|
||||
namespace Poco { class Logger; }
|
||||
|
||||
/// Extract format string from a string literal and constructs consteval fmt::format_string
|
||||
template <typename... Args>
|
||||
struct FormatStringHelperImpl
|
||||
{
|
||||
std::string_view message_format_string;
|
||||
fmt::format_string<Args...> fmt_str;
|
||||
template<typename T>
|
||||
consteval FormatStringHelperImpl(T && str) : message_format_string(tryGetStaticFormatString(str)), fmt_str(std::forward<T>(str)) {}
|
||||
template<typename T>
|
||||
FormatStringHelperImpl(fmt::basic_runtime<T> && str) : message_format_string(), fmt_str(std::forward<fmt::basic_runtime<T>>(str)) {}
|
||||
|
||||
PreformattedMessage format(Args && ...args) const
|
||||
{
|
||||
return PreformattedMessage{fmt::format(fmt_str, std::forward<Args...>(args)...), message_format_string};
|
||||
}
|
||||
};
|
||||
|
||||
template <typename... Args>
|
||||
using FormatStringHelper = FormatStringHelperImpl<std::type_identity_t<Args>...>;
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -48,6 +28,17 @@ public:
|
||||
|
||||
Exception() = default;
|
||||
|
||||
Exception(const PreformattedMessage & msg, int code): Exception(msg.text, code)
|
||||
{
|
||||
message_format_string = msg.format_string;
|
||||
}
|
||||
|
||||
Exception(PreformattedMessage && msg, int code): Exception(std::move(msg.text), code)
|
||||
{
|
||||
message_format_string = msg.format_string;
|
||||
}
|
||||
|
||||
protected:
|
||||
// used to remove the sensitive information from exceptions if query_masking_rules is configured
|
||||
struct MessageMasked
|
||||
{
|
||||
@ -62,11 +53,16 @@ public:
|
||||
// delegating constructor to mask sensitive information from the message
|
||||
Exception(const std::string & msg, int code, bool remote_ = false): Exception(MessageMasked(msg), code, remote_) {}
|
||||
Exception(std::string && msg, int code, bool remote_ = false): Exception(MessageMasked(std::move(msg)), code, remote_) {}
|
||||
Exception(PreformattedMessage && msg, int code): Exception(std::move(msg.message), code)
|
||||
|
||||
public:
|
||||
/// This creator is for exceptions that should format a message using fmt::format from the variadic ctor Exception(code, fmt, ...),
|
||||
/// but were not rewritten yet. It will be removed.
|
||||
static Exception createDeprecated(const std::string & msg, int code, bool remote_ = false)
|
||||
{
|
||||
message_format_string = msg.format_string;
|
||||
return Exception(msg, code, remote_);
|
||||
}
|
||||
|
||||
/// Message must be a compile-time constant
|
||||
template<typename T, typename = std::enable_if_t<std::is_convertible_v<T, String>>>
|
||||
Exception(int code, T && message)
|
||||
: Exception(message, code)
|
||||
@ -74,9 +70,11 @@ public:
|
||||
message_format_string = tryGetStaticFormatString(message);
|
||||
}
|
||||
|
||||
template<> Exception(int code, const String & message) : Exception(message, code) {}
|
||||
template<> Exception(int code, String & message) : Exception(message, code) {}
|
||||
template<> Exception(int code, String && message) : Exception(std::move(message), code) {}
|
||||
/// These creators are for messages that were received by network or generated by a third-party library in runtime.
|
||||
/// Please use a constructor for all other cases.
|
||||
static Exception createRuntime(int code, const String & message) { return Exception(message, code); }
|
||||
static Exception createRuntime(int code, String & message) { return Exception(message, code); }
|
||||
static Exception createRuntime(int code, String && message) { return Exception(std::move(message), code); }
|
||||
|
||||
// Format message with fmt::format, like the logging functions.
|
||||
template <typename... Args>
|
||||
@ -167,11 +165,9 @@ private:
|
||||
/// more convenient calculation of problem line number.
|
||||
class ParsingException : public Exception
|
||||
{
|
||||
ParsingException(const std::string & msg, int code);
|
||||
public:
|
||||
ParsingException();
|
||||
ParsingException(const std::string & msg, int code);
|
||||
ParsingException(int code, const std::string & message);
|
||||
ParsingException(int code, std::string && message) : Exception(message, code) {}
|
||||
|
||||
// Format message with fmt::format, like the logging functions.
|
||||
template <typename... Args>
|
||||
|
||||
7
src/Common/LoggingFormatStringHelpers.cpp
Normal file
7
src/Common/LoggingFormatStringHelpers.cpp
Normal file
@ -0,0 +1,7 @@
|
||||
#include <Common/LoggingFormatStringHelpers.h>
|
||||
|
||||
[[noreturn]] void functionThatFailsCompilationOfConstevalFunctions(const char * error)
|
||||
{
|
||||
throw std::runtime_error(error);
|
||||
}
|
||||
|
||||
@ -2,24 +2,70 @@
|
||||
#include <base/defines.h>
|
||||
#include <fmt/format.h>
|
||||
|
||||
struct PreformattedMessage;
|
||||
consteval void formatStringCheckArgsNumImpl(std::string_view str, size_t nargs);
|
||||
template <typename T> constexpr std::string_view tryGetStaticFormatString(T && x);
|
||||
|
||||
/// Extract format string from a string literal and constructs consteval fmt::format_string
|
||||
template <typename... Args>
|
||||
struct FormatStringHelperImpl
|
||||
{
|
||||
std::string_view message_format_string;
|
||||
fmt::format_string<Args...> fmt_str;
|
||||
template<typename T>
|
||||
consteval FormatStringHelperImpl(T && str) : message_format_string(tryGetStaticFormatString(str)), fmt_str(std::forward<T>(str))
|
||||
{
|
||||
formatStringCheckArgsNumImpl(message_format_string, sizeof...(Args));
|
||||
}
|
||||
template<typename T>
|
||||
FormatStringHelperImpl(fmt::basic_runtime<T> && str) : message_format_string(), fmt_str(std::forward<fmt::basic_runtime<T>>(str)) {}
|
||||
|
||||
PreformattedMessage format(Args && ...args) const;
|
||||
};
|
||||
|
||||
template <typename... Args>
|
||||
using FormatStringHelper = FormatStringHelperImpl<std::type_identity_t<Args>...>;
|
||||
|
||||
/// Saves a format string for already formatted message
|
||||
struct PreformattedMessage
|
||||
{
|
||||
String message;
|
||||
std::string text;
|
||||
std::string_view format_string;
|
||||
|
||||
operator const String & () const { return message; }
|
||||
operator String () && { return std::move(message); }
|
||||
template <typename... Args>
|
||||
static PreformattedMessage create(FormatStringHelper<Args...> fmt, Args &&... args);
|
||||
|
||||
operator const std::string & () const { return text; }
|
||||
operator std::string () && { return std::move(text); }
|
||||
operator fmt::format_string<> () const { UNREACHABLE(); }
|
||||
};
|
||||
|
||||
template <typename... Args>
|
||||
PreformattedMessage FormatStringHelperImpl<Args...>::format(Args && ...args) const
|
||||
{
|
||||
return PreformattedMessage{fmt::format(fmt_str, std::forward<Args>(args)...), message_format_string};
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
PreformattedMessage PreformattedMessage::create(FormatStringHelper<Args...> fmt, Args && ...args)
|
||||
{
|
||||
return fmt.format(std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template<typename T> struct is_fmt_runtime : std::false_type {};
|
||||
template<typename T> struct is_fmt_runtime<fmt::basic_runtime<T>> : std::true_type {};
|
||||
|
||||
template <typename T> constexpr std::string_view tryGetStaticFormatString(T && x)
|
||||
{
|
||||
/// Failure of this asserting indicates that something went wrong during type deduction.
|
||||
/// For example, a string literal was implicitly converted to std::string. It should not happen.
|
||||
/// Format string for an exception or log message must be a string literal (compile-time constant).
|
||||
/// Failure of this assertion may indicate one of the following issues:
|
||||
/// - A message was already formatted into std::string before passing to Exception(...) or LOG_XXXXX(...).
|
||||
/// Please use variadic constructor of Exception.
|
||||
/// Consider using PreformattedMessage or LogToStr if you want to avoid double formatting and/or copy-paste.
|
||||
/// - A string literal was converted to std::string (or const char *).
|
||||
/// - Use Exception::createRuntime or fmt::runtime if there's no format string
|
||||
/// and a message is generated in runtime by a third-party library
|
||||
/// or deserialized from somewhere.
|
||||
static_assert(!std::is_same_v<std::string, std::decay_t<T>>);
|
||||
|
||||
if constexpr (is_fmt_runtime<std::decay_t<T>>::value)
|
||||
@ -53,3 +99,60 @@ template <typename T, typename... Ts> constexpr auto firstArg(T && x, Ts &&...)
|
||||
/// For implicit conversion of fmt::basic_runtime<> to char* for std::string ctor
|
||||
template <typename T, typename... Ts> constexpr auto firstArg(fmt::basic_runtime<T> && data, Ts &&...) { return data.str.data(); }
|
||||
|
||||
consteval ssize_t formatStringCountArgsNum(const char * const str, size_t len)
|
||||
{
|
||||
/// It does not count named args, but we don't use them
|
||||
size_t cnt = 0;
|
||||
size_t i = 0;
|
||||
while (i + 1 < len)
|
||||
{
|
||||
if (str[i] == '{' && str[i + 1] == '}')
|
||||
{
|
||||
i += 2;
|
||||
cnt += 1;
|
||||
}
|
||||
else if (str[i] == '{')
|
||||
{
|
||||
/// Ignore checks for complex formatting like "{:.3f}"
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
return cnt;
|
||||
}
|
||||
|
||||
[[noreturn]] void functionThatFailsCompilationOfConstevalFunctions(const char * error);
|
||||
|
||||
/// fmt::format checks that there are enough arguments, but ignores extra arguments (e.g. fmt::format("{}", 1, 2) compiles)
|
||||
/// This function will fail to compile if the number of "{}" substitutions does not exactly match
|
||||
consteval void formatStringCheckArgsNumImpl(std::string_view str, size_t nargs)
|
||||
{
|
||||
if (str.empty())
|
||||
return;
|
||||
ssize_t cnt = formatStringCountArgsNum(str.data(), str.size());
|
||||
if (0 <= cnt && cnt != nargs)
|
||||
functionThatFailsCompilationOfConstevalFunctions("unexpected number of arguments in a format string");
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
struct CheckArgsNumHelperImpl
|
||||
{
|
||||
template<typename T>
|
||||
consteval CheckArgsNumHelperImpl(T && str)
|
||||
{
|
||||
formatStringCheckArgsNumImpl(tryGetStaticFormatString(str), sizeof...(Args));
|
||||
}
|
||||
|
||||
/// No checks for fmt::runtime and PreformattedMessage
|
||||
template<typename T> CheckArgsNumHelperImpl(fmt::basic_runtime<T> &&) {}
|
||||
template<> CheckArgsNumHelperImpl(PreformattedMessage &) {}
|
||||
template<> CheckArgsNumHelperImpl(const PreformattedMessage &) {}
|
||||
template<> CheckArgsNumHelperImpl(PreformattedMessage &&) {}
|
||||
|
||||
};
|
||||
|
||||
template <typename... Args> using CheckArgsNumHelper = CheckArgsNumHelperImpl<std::type_identity_t<Args>...>;
|
||||
template <typename... Args> void formatStringCheckArgsNum(CheckArgsNumHelper<Args...>, Args &&...) {}
|
||||
|
||||
@ -187,7 +187,7 @@ public:
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::BAD_ARGUMENTS,
|
||||
"Value with key `{}` is used twice in the SET query",
|
||||
"Value with key `{}` is used twice in the SET query (collection name: {})",
|
||||
name, query.collection_name);
|
||||
}
|
||||
}
|
||||
|
||||
@ -211,9 +211,8 @@ PoolWithFailoverBase<TNestedPool>::get(size_t max_ignored_errors, bool fallback_
|
||||
max_ignored_errors, fallback_to_stale_replicas,
|
||||
try_get_entry, get_priority);
|
||||
if (results.empty() || results[0].entry.isNull())
|
||||
throw DB::Exception(
|
||||
"PoolWithFailoverBase::getMany() returned less than min_entries entries.",
|
||||
DB::ErrorCodes::LOGICAL_ERROR);
|
||||
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR,
|
||||
"PoolWithFailoverBase::getMany() returned less than min_entries entries.");
|
||||
return results[0].entry;
|
||||
}
|
||||
|
||||
@ -320,10 +319,8 @@ PoolWithFailoverBase<TNestedPool>::getMany(
|
||||
try_results.resize(up_to_date_count);
|
||||
}
|
||||
else
|
||||
throw DB::Exception(
|
||||
"Could not find enough connections to up-to-date replicas. Got: " + std::to_string(up_to_date_count)
|
||||
+ ", needed: " + std::to_string(min_entries),
|
||||
DB::ErrorCodes::ALL_REPLICAS_ARE_STALE);
|
||||
throw DB::Exception(DB::ErrorCodes::ALL_REPLICAS_ARE_STALE,
|
||||
"Could not find enough connections to up-to-date replicas. Got: {}, needed: {}", up_to_date_count, max_entries);
|
||||
|
||||
return try_results;
|
||||
}
|
||||
|
||||
@ -62,10 +62,10 @@ public:
|
||||
, replacement(replacement_string)
|
||||
{
|
||||
if (!regexp.ok())
|
||||
throw DB::Exception(
|
||||
"SensitiveDataMasker: cannot compile re2: " + regexp_string_ + ", error: " + regexp.error()
|
||||
+ ". Look at https://github.com/google/re2/wiki/Syntax for reference.",
|
||||
DB::ErrorCodes::CANNOT_COMPILE_REGEXP);
|
||||
throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP,
|
||||
"SensitiveDataMasker: cannot compile re2: {}, error: {}. "
|
||||
"Look at https://github.com/google/re2/wiki/Syntax for reference.",
|
||||
regexp_string_, regexp.error());
|
||||
}
|
||||
|
||||
uint64_t apply(std::string & data) const
|
||||
|
||||
@ -100,7 +100,7 @@ size_t TLDListsHolder::parseAndAddTldList(const std::string & name, const std::s
|
||||
tld_list_tmp.emplace(line, TLDType::TLD_REGULAR);
|
||||
}
|
||||
if (!in.eof())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not all list had been read", name);
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not all list had been read: {}", name);
|
||||
|
||||
TLDList tld_list(tld_list_tmp.size());
|
||||
for (const auto & [host, type] : tld_list_tmp)
|
||||
|
||||
@ -58,7 +58,7 @@ UInt64 Throttler::add(size_t amount)
|
||||
}
|
||||
|
||||
if (limit && count_value > limit)
|
||||
throw Exception(limit_exceeded_exception_message + std::string(" Maximum: ") + toString(limit), ErrorCodes::LIMIT_EXCEEDED);
|
||||
throw Exception::createDeprecated(limit_exceeded_exception_message + std::string(" Maximum: ") + toString(limit), ErrorCodes::LIMIT_EXCEEDED);
|
||||
|
||||
/// Wait unless there is positive amount of tokens - throttling
|
||||
Int64 sleep_time = 0;
|
||||
|
||||
@ -41,7 +41,7 @@ To assert_cast(From && from)
|
||||
}
|
||||
catch (const std::exception & e)
|
||||
{
|
||||
throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
|
||||
throw DB::Exception::createDeprecated(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Bad cast from type {} to {}",
|
||||
|
||||
@ -312,39 +312,32 @@ bool exists(const std::string & path)
|
||||
|
||||
bool canRead(const std::string & path)
|
||||
{
|
||||
struct stat st;
|
||||
if (stat(path.c_str(), &st) == 0)
|
||||
{
|
||||
if (st.st_uid == geteuid())
|
||||
return (st.st_mode & S_IRUSR) != 0;
|
||||
else if (st.st_gid == getegid())
|
||||
return (st.st_mode & S_IRGRP) != 0;
|
||||
else
|
||||
return (st.st_mode & S_IROTH) != 0 || geteuid() == 0;
|
||||
}
|
||||
int err = faccessat(AT_FDCWD, path.c_str(), R_OK, AT_EACCESS);
|
||||
if (err == 0)
|
||||
return true;
|
||||
if (errno == EACCES)
|
||||
return false;
|
||||
DB::throwFromErrnoWithPath("Cannot check read access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
|
||||
}
|
||||
|
||||
bool canWrite(const std::string & path)
|
||||
{
|
||||
struct stat st;
|
||||
if (stat(path.c_str(), &st) == 0)
|
||||
{
|
||||
if (st.st_uid == geteuid())
|
||||
return (st.st_mode & S_IWUSR) != 0;
|
||||
else if (st.st_gid == getegid())
|
||||
return (st.st_mode & S_IWGRP) != 0;
|
||||
else
|
||||
return (st.st_mode & S_IWOTH) != 0 || geteuid() == 0;
|
||||
}
|
||||
int err = faccessat(AT_FDCWD, path.c_str(), W_OK, AT_EACCESS);
|
||||
if (err == 0)
|
||||
return true;
|
||||
if (errno == EACCES)
|
||||
return false;
|
||||
DB::throwFromErrnoWithPath("Cannot check write access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
|
||||
}
|
||||
|
||||
bool canExecute(const std::string & path)
|
||||
{
|
||||
if (exists(path))
|
||||
return faccessat(AT_FDCWD, path.c_str(), X_OK, AT_EACCESS) == 0;
|
||||
DB::throwFromErrnoWithPath("Cannot check execute access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
|
||||
int err = faccessat(AT_FDCWD, path.c_str(), X_OK, AT_EACCESS);
|
||||
if (err == 0)
|
||||
return true;
|
||||
if (errno == EACCES)
|
||||
return false;
|
||||
DB::throwFromErrnoWithPath("Cannot check write access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
|
||||
}
|
||||
|
||||
time_t getModificationTime(const std::string & path)
|
||||
|
||||
@ -57,6 +57,7 @@ namespace
|
||||
if (_is_clients_log || _logger->is((PRIORITY))) \
|
||||
{ \
|
||||
std::string formatted_message = numArgs(__VA_ARGS__) > 1 ? fmt::format(__VA_ARGS__) : firstArg(__VA_ARGS__); \
|
||||
formatStringCheckArgsNum(__VA_ARGS__); \
|
||||
if (auto _channel = _logger->getChannel()) \
|
||||
{ \
|
||||
std::string file_function; \
|
||||
|
||||
@ -171,9 +171,8 @@ TEST(Common, RWLockDeadlock)
|
||||
auto holder2 = lock2->getLock(RWLockImpl::Read, "q1", std::chrono::milliseconds(100));
|
||||
if (!holder2)
|
||||
{
|
||||
throw Exception(
|
||||
"Locking attempt timed out! Possible deadlock avoided. Client should retry.",
|
||||
ErrorCodes::DEADLOCK_AVOIDED);
|
||||
throw Exception(ErrorCodes::DEADLOCK_AVOIDED,
|
||||
"Locking attempt timed out! Possible deadlock avoided. Client should retry.");
|
||||
}
|
||||
}
|
||||
catch (const Exception & e)
|
||||
@ -202,9 +201,8 @@ TEST(Common, RWLockDeadlock)
|
||||
auto holder1 = lock1->getLock(RWLockImpl::Read, "q3", std::chrono::milliseconds(100));
|
||||
if (!holder1)
|
||||
{
|
||||
throw Exception(
|
||||
"Locking attempt timed out! Possible deadlock avoided. Client should retry.",
|
||||
ErrorCodes::DEADLOCK_AVOIDED);
|
||||
throw Exception(ErrorCodes::DEADLOCK_AVOIDED,
|
||||
"Locking attempt timed out! Possible deadlock avoided. Client should retry.");
|
||||
}
|
||||
}
|
||||
catch (const Exception & e)
|
||||
|
||||
@ -37,7 +37,7 @@ To typeid_cast(From & from)
|
||||
}
|
||||
catch (const std::exception & e)
|
||||
{
|
||||
throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
|
||||
throw DB::Exception::createDeprecated(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Bad cast from type {} to {}",
|
||||
@ -58,7 +58,7 @@ To typeid_cast(From * from)
|
||||
}
|
||||
catch (const std::exception & e)
|
||||
{
|
||||
throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
|
||||
throw DB::Exception::createDeprecated(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
@ -93,6 +93,6 @@ To typeid_cast(const std::shared_ptr<From> & from)
|
||||
}
|
||||
catch (const std::exception & e)
|
||||
{
|
||||
throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
|
||||
throw DB::Exception::createDeprecated(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
@ -86,7 +86,7 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c
|
||||
{
|
||||
message << ". The mismatch is caused by single bit flip in data block at byte " << (bit_pos / 8) << ", bit " << (bit_pos % 8) << ". "
|
||||
<< message_hardware_failure;
|
||||
throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH);
|
||||
throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH);
|
||||
}
|
||||
|
||||
flip_bit(tmp_data, bit_pos); /// Restore
|
||||
@ -101,10 +101,10 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c
|
||||
{
|
||||
message << ". The mismatch is caused by single bit flip in checksum. "
|
||||
<< message_hardware_failure;
|
||||
throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH);
|
||||
throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH);
|
||||
}
|
||||
|
||||
throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH);
|
||||
throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH);
|
||||
}
|
||||
|
||||
static void readHeaderAndGetCodecAndSize(
|
||||
|
||||
@ -141,7 +141,7 @@ size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, Encryption
|
||||
reinterpret_cast<const uint8_t*>(key.data()), key.size(),
|
||||
tag_size, nullptr);
|
||||
if (!ok_init)
|
||||
throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
/// encrypt data using context and given nonce.
|
||||
size_t out_len;
|
||||
@ -152,7 +152,7 @@ size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, Encryption
|
||||
reinterpret_cast<const uint8_t *>(plaintext.data()), plaintext.size(),
|
||||
nullptr, 0);
|
||||
if (!ok_open)
|
||||
throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
return out_len;
|
||||
}
|
||||
@ -171,7 +171,7 @@ size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod m
|
||||
reinterpret_cast<const uint8_t*>(key.data()), key.size(),
|
||||
tag_size, nullptr);
|
||||
if (!ok_init)
|
||||
throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
/// decrypt data using given nonce
|
||||
size_t out_len;
|
||||
@ -182,7 +182,7 @@ size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod m
|
||||
reinterpret_cast<const uint8_t *>(ciphertext.data()), ciphertext.size(),
|
||||
nullptr, 0);
|
||||
if (!ok_open)
|
||||
throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
return out_len;
|
||||
}
|
||||
|
||||
@ -39,6 +39,7 @@ protected:
|
||||
|
||||
bool isCompression() const override { return true; }
|
||||
bool isGenericCompression() const override { return false; }
|
||||
bool isFloatingPointTimeSeriesCodec() const override { return true; }
|
||||
|
||||
private:
|
||||
static constexpr UInt32 HEADER_SIZE = 2;
|
||||
|
||||
@ -122,6 +122,7 @@ protected:
|
||||
|
||||
bool isCompression() const override { return true; }
|
||||
bool isGenericCompression() const override { return false; }
|
||||
bool isFloatingPointTimeSeriesCodec() const override { return true; }
|
||||
|
||||
private:
|
||||
const UInt8 data_bytes_size;
|
||||
@ -442,14 +443,14 @@ void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 sourc
|
||||
void registerCodecGorilla(CompressionCodecFactory & factory)
|
||||
{
|
||||
UInt8 method_code = static_cast<UInt8>(CompressionMethodByte::Gorilla);
|
||||
factory.registerCompressionCodecWithType("Gorilla", method_code,
|
||||
[&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
|
||||
auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
|
||||
{
|
||||
if (arguments)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Gorilla does not accept any arguments");
|
||||
|
||||
UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0;
|
||||
return std::make_shared<CompressionCodecGorilla>(data_bytes_size);
|
||||
});
|
||||
};
|
||||
factory.registerCompressionCodecWithType("Gorilla", method_code, codec_builder);
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,7 +15,9 @@
|
||||
#include <DataTypes/NestedUtils.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeMap.h>
|
||||
#include <DataTypes/DataTypeNested.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
|
||||
@ -51,6 +53,28 @@ void CompressionCodecFactory::validateCodec(
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
bool innerDataTypeIsFloat(const DataTypePtr & type)
|
||||
{
|
||||
if (isFloat(type))
|
||||
return true;
|
||||
if (const DataTypeNullable * type_nullable = typeid_cast<const DataTypeNullable *>(type.get()))
|
||||
return innerDataTypeIsFloat(type_nullable->getNestedType());
|
||||
if (const DataTypeArray * type_array = typeid_cast<const DataTypeArray *>(type.get()))
|
||||
return innerDataTypeIsFloat(type_array->getNestedType());
|
||||
if (const DataTypeTuple * type_tuple = typeid_cast<const DataTypeTuple *>(type.get()))
|
||||
{
|
||||
for (const auto & subtype : type_tuple->getElements())
|
||||
if (innerDataTypeIsFloat(subtype))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||
const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs) const
|
||||
@ -59,15 +83,16 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||
{
|
||||
ASTPtr codecs_descriptions = std::make_shared<ASTExpressionList>();
|
||||
|
||||
bool is_compression = false;
|
||||
bool has_none = false;
|
||||
bool with_compression_codec = false;
|
||||
bool with_none_codec = false;
|
||||
bool with_floating_point_timeseries_codec = false;
|
||||
std::optional<size_t> generic_compression_codec_pos;
|
||||
std::set<size_t> encryption_codecs;
|
||||
std::set<size_t> encryption_codecs_pos;
|
||||
|
||||
bool can_substitute_codec_arguments = true;
|
||||
for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i)
|
||||
{
|
||||
const auto & inner_codec_ast = func->arguments->children[i];
|
||||
const ASTPtr & inner_codec_ast = func->arguments->children[i];
|
||||
String codec_family_name;
|
||||
ASTPtr codec_arguments;
|
||||
if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
|
||||
@ -136,21 +161,22 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||
codecs_descriptions->children.emplace_back(result_codec->getCodecDesc());
|
||||
}
|
||||
|
||||
is_compression |= result_codec->isCompression();
|
||||
has_none |= result_codec->isNone();
|
||||
with_compression_codec |= result_codec->isCompression();
|
||||
with_none_codec |= result_codec->isNone();
|
||||
with_floating_point_timeseries_codec |= result_codec->isFloatingPointTimeSeriesCodec();
|
||||
|
||||
if (!generic_compression_codec_pos && result_codec->isGenericCompression())
|
||||
generic_compression_codec_pos = i;
|
||||
|
||||
if (result_codec->isEncryption())
|
||||
encryption_codecs.insert(i);
|
||||
encryption_codecs_pos.insert(i);
|
||||
}
|
||||
|
||||
String codec_description = queryToString(codecs_descriptions);
|
||||
|
||||
if (sanity_check)
|
||||
{
|
||||
if (codecs_descriptions->children.size() > 1 && has_none)
|
||||
if (codecs_descriptions->children.size() > 1 && with_none_codec)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"It does not make sense to have codec NONE along with other compression codecs: {}. "
|
||||
"(Note: you can enable setting 'allow_suspicious_codecs' to skip this check).",
|
||||
@ -159,7 +185,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||
/// Allow to explicitly specify single NONE codec if user don't want any compression.
|
||||
/// But applying other transformations solely without compression (e.g. Delta) does not make sense.
|
||||
/// It's okay to apply encryption codecs solely without anything else.
|
||||
if (!is_compression && !has_none && encryption_codecs.size() != codecs_descriptions->children.size())
|
||||
if (!with_compression_codec && !with_none_codec && encryption_codecs_pos.size() != codecs_descriptions->children.size())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Compression codec {} does not compress anything. "
|
||||
"You may want to add generic compression algorithm after other transformations, like: {}, LZ4. "
|
||||
"(Note: you can enable setting 'allow_suspicious_codecs' to skip this check).",
|
||||
@ -167,17 +193,25 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||
|
||||
/// It does not make sense to apply any non-encryption codecs
|
||||
/// after encryption one.
|
||||
if (!encryption_codecs.empty() &&
|
||||
*encryption_codecs.begin() != codecs_descriptions->children.size() - encryption_codecs.size())
|
||||
if (!encryption_codecs_pos.empty() &&
|
||||
*encryption_codecs_pos.begin() != codecs_descriptions->children.size() - encryption_codecs_pos.size())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The combination of compression codecs {} is meaningless, "
|
||||
"because it does not make sense to apply any non-post-processing codecs after "
|
||||
"post-processing ones. (Note: you can enable setting 'allow_suspicious_codecs' "
|
||||
"to skip this check).", codec_description);
|
||||
|
||||
/// Floating-point time series codecs are not supposed to compress non-floating-point data
|
||||
if (with_floating_point_timeseries_codec &&
|
||||
column_type && !innerDataTypeIsFloat(column_type))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"The combination of compression codecs {} is meaningless,"
|
||||
" because it does not make sense to apply a floating-point time series codec to non-floating-point columns"
|
||||
" (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", codec_description);
|
||||
|
||||
/// It does not make sense to apply any transformations after generic compression algorithm
|
||||
/// So, generic compression can be only one and only at the end.
|
||||
if (generic_compression_codec_pos &&
|
||||
*generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs.size())
|
||||
*generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs_pos.size())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The combination of compression codecs {} is meaningless, "
|
||||
"because it does not make sense to apply any transformations after generic "
|
||||
"compression algorithm. (Note: you can enable setting 'allow_suspicious_codecs' "
|
||||
|
||||
@ -99,6 +99,9 @@ public:
|
||||
/// If it is a post-processing codec such as encryption. Usually it does not make sense to apply non-post-processing codecs after this.
|
||||
virtual bool isEncryption() const { return false; }
|
||||
|
||||
/// If it is a specialized codec for floating-point time series. Applying it to non-floating point data is suspicious.
|
||||
virtual bool isFloatingPointTimeSeriesCodec() const { return false; }
|
||||
|
||||
/// It is a codec available only for evaluation purposes and not meant to be used in production.
|
||||
/// It will not be allowed to use unless the user will turn off the safety switch.
|
||||
virtual bool isExperimental() const { return false; }
|
||||
|
||||
@ -529,6 +529,13 @@ public:
|
||||
|
||||
TEST_P(CodecTest, TranscodingWithDataType)
|
||||
{
|
||||
/// Gorilla can only be applied to floating point columns
|
||||
bool codec_is_gorilla = std::get<0>(GetParam()).codec_statement.find("Gorilla") != std::string::npos;
|
||||
WhichDataType which(std::get<1>(GetParam()).data_type.get());
|
||||
bool data_is_float = which.isFloat();
|
||||
if (codec_is_gorilla && !data_is_float)
|
||||
GTEST_SKIP() << "Skipping Gorilla-compressed non-float column";
|
||||
|
||||
const auto codec = makeCodec(CODEC_WITH_DATA_TYPE);
|
||||
testTranscoding(*codec);
|
||||
}
|
||||
@ -1204,68 +1211,20 @@ auto DDperformanceTestSequence()
|
||||
}
|
||||
|
||||
// prime numbers in ascending order with some random repetitions hit all the cases of Gorilla.
|
||||
auto PrimesWithMultiplierGenerator = [](int multiplier = 1)
|
||||
{
|
||||
return [multiplier](auto i)
|
||||
{
|
||||
static const int vals[] = {
|
||||
2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43,
|
||||
47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103,
|
||||
107, 107, 109, 113, 113, 127, 127, 127
|
||||
};
|
||||
static const size_t count = sizeof(vals)/sizeof(vals[0]);
|
||||
|
||||
return static_cast<UInt64>(vals[i % count]) * multiplier;
|
||||
};
|
||||
};
|
||||
|
||||
template <typename ValueType>
|
||||
auto GCompatibilityTestSequence()
|
||||
{
|
||||
// Also multiply result by some factor to test large values on types that can hold those.
|
||||
return generateSeq<ValueType>(G(PrimesWithMultiplierGenerator(intExp10(sizeof(ValueType)))), 0, 42);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(Gorilla,
|
||||
CodecTestCompatibility,
|
||||
::testing::Combine(
|
||||
::testing::Values(Codec("Gorilla")),
|
||||
::testing::ValuesIn(std::initializer_list<std::tuple<CodecTestSequence, std::string>>{
|
||||
{
|
||||
GCompatibilityTestSequence<Int8>(),
|
||||
BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a")
|
||||
},
|
||||
{
|
||||
GCompatibilityTestSequence<UInt8>(),
|
||||
BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a")
|
||||
},
|
||||
{
|
||||
GCompatibilityTestSequence<Int16>(),
|
||||
BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4")
|
||||
},
|
||||
{
|
||||
GCompatibilityTestSequence<UInt16>(),
|
||||
BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4")
|
||||
},
|
||||
{
|
||||
GCompatibilityTestSequence<Int32>(),
|
||||
BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4")
|
||||
},
|
||||
{
|
||||
GCompatibilityTestSequence<UInt32>(),
|
||||
BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4")
|
||||
},
|
||||
{
|
||||
GCompatibilityTestSequence<Int64>(),
|
||||
BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10")
|
||||
},
|
||||
{
|
||||
GCompatibilityTestSequence<UInt64>(),
|
||||
BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10")
|
||||
},
|
||||
})
|
||||
)
|
||||
);
|
||||
// auto PrimesWithMultiplierGenerator = [](int multiplier = 1)
|
||||
// {
|
||||
// return [multiplier](auto i)
|
||||
// {
|
||||
// static const int vals[] = {
|
||||
// 2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43,
|
||||
// 47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103,
|
||||
// 107, 107, 109, 113, 113, 127, 127, 127
|
||||
// };
|
||||
// static const size_t count = sizeof(vals)/sizeof(vals[0]);
|
||||
//
|
||||
// return static_cast<UInt64>(vals[i % count]) * multiplier;
|
||||
// };
|
||||
// };
|
||||
|
||||
// These 'tests' try to measure performance of encoding and decoding and hence only make sense to be run locally,
|
||||
// also they require pretty big data to run against and generating this data slows down startup of unit test process.
|
||||
|
||||
@ -29,11 +29,13 @@ namespace ErrorCodes
|
||||
extern const int AMBIGUOUS_COLUMN_NAME;
|
||||
}
|
||||
|
||||
template <typename ReturnType>
|
||||
static ReturnType onError(const std::string & message [[maybe_unused]], int code [[maybe_unused]])
|
||||
template <typename ReturnType, typename... FmtArgs>
|
||||
static ReturnType onError(int code [[maybe_unused]],
|
||||
FormatStringHelper<FmtArgs...> fmt_string [[maybe_unused]],
|
||||
FmtArgs && ...fmt_args [[maybe_unused]])
|
||||
{
|
||||
if constexpr (std::is_same_v<ReturnType, void>)
|
||||
throw Exception(message, code);
|
||||
throw Exception(code, std::move(fmt_string), std::forward<FmtArgs>(fmt_args)...);
|
||||
else
|
||||
return false;
|
||||
}
|
||||
@ -44,13 +46,13 @@ static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, con
|
||||
std::string_view context_description, bool allow_materialize, int code)
|
||||
{
|
||||
if (actual.name != expected.name)
|
||||
return onError<ReturnType>("Block structure mismatch in " + std::string(context_description) + " stream: different names of columns:\n"
|
||||
+ actual.dumpStructure() + "\n" + expected.dumpStructure(), code);
|
||||
return onError<ReturnType>(code, "Block structure mismatch in {} stream: different names of columns:\n{}\n{}",
|
||||
context_description, actual.dumpStructure(), expected.dumpStructure());
|
||||
|
||||
if ((actual.type && !expected.type) || (!actual.type && expected.type)
|
||||
|| (actual.type && expected.type && !actual.type->equals(*expected.type)))
|
||||
return onError<ReturnType>("Block structure mismatch in " + std::string(context_description) + " stream: different types:\n"
|
||||
+ actual.dumpStructure() + "\n" + expected.dumpStructure(), code);
|
||||
return onError<ReturnType>(code, "Block structure mismatch in {} stream: different types:\n{}\n{}",
|
||||
context_description, actual.dumpStructure(), expected.dumpStructure());
|
||||
|
||||
if (!actual.column || !expected.column)
|
||||
return ReturnType(true);
|
||||
@ -74,22 +76,18 @@ static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, con
|
||||
if (actual_column_maybe_agg && expected_column_maybe_agg)
|
||||
{
|
||||
if (!actual_column_maybe_agg->getAggregateFunction()->haveSameStateRepresentation(*expected_column_maybe_agg->getAggregateFunction()))
|
||||
return onError<ReturnType>(
|
||||
fmt::format(
|
||||
return onError<ReturnType>(code,
|
||||
"Block structure mismatch in {} stream: different columns:\n{}\n{}",
|
||||
context_description,
|
||||
actual.dumpStructure(),
|
||||
expected.dumpStructure()),
|
||||
code);
|
||||
expected.dumpStructure());
|
||||
}
|
||||
else if (actual_column->getName() != expected.column->getName())
|
||||
return onError<ReturnType>(
|
||||
fmt::format(
|
||||
return onError<ReturnType>(code,
|
||||
"Block structure mismatch in {} stream: different columns:\n{}\n{}",
|
||||
context_description,
|
||||
actual.dumpStructure(),
|
||||
expected.dumpStructure()),
|
||||
code);
|
||||
expected.dumpStructure());
|
||||
|
||||
if (isColumnConst(*actual.column) && isColumnConst(*expected.column)
|
||||
&& !actual.column->empty() && !expected.column->empty()) /// don't check values in empty columns
|
||||
@ -98,14 +96,12 @@ static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, con
|
||||
Field expected_value = assert_cast<const ColumnConst &>(*expected.column).getField();
|
||||
|
||||
if (actual_value != expected_value)
|
||||
return onError<ReturnType>(
|
||||
fmt::format(
|
||||
return onError<ReturnType>(code,
|
||||
"Block structure mismatch in {} stream: different values of constants in column '{}': actual: {}, expected: {}",
|
||||
context_description,
|
||||
actual.name,
|
||||
applyVisitor(FieldVisitorToString(), actual_value),
|
||||
applyVisitor(FieldVisitorToString(), expected_value)),
|
||||
code);
|
||||
applyVisitor(FieldVisitorToString(), expected_value));
|
||||
}
|
||||
|
||||
return ReturnType(true);
|
||||
@ -117,8 +113,8 @@ static ReturnType checkBlockStructure(const Block & lhs, const Block & rhs, std:
|
||||
{
|
||||
size_t columns = rhs.columns();
|
||||
if (lhs.columns() != columns)
|
||||
return onError<ReturnType>("Block structure mismatch in " + std::string(context_description) + " stream: different number of columns:\n"
|
||||
+ lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR);
|
||||
return onError<ReturnType>(ErrorCodes::LOGICAL_ERROR, "Block structure mismatch in {} stream: different number of columns:\n{}\n{}",
|
||||
context_description, lhs.dumpStructure(), rhs.dumpStructure());
|
||||
|
||||
for (size_t i = 0; i < columns; ++i)
|
||||
{
|
||||
|
||||
@ -95,7 +95,7 @@ void MySQLClient::handshake()
|
||||
packet_endpoint->resetSequenceId();
|
||||
|
||||
if (packet_response.getType() == PACKET_ERR)
|
||||
throw Exception(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
|
||||
throw Exception::createDeprecated(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
|
||||
else if (packet_response.getType() == PACKET_AUTH_SWITCH)
|
||||
throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Access denied for user {}", user);
|
||||
}
|
||||
@ -110,7 +110,7 @@ void MySQLClient::writeCommand(char command, String query)
|
||||
switch (packet_response.getType())
|
||||
{
|
||||
case PACKET_ERR:
|
||||
throw Exception(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
|
||||
throw Exception::createDeprecated(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
|
||||
case PACKET_OK:
|
||||
break;
|
||||
default:
|
||||
@ -128,7 +128,7 @@ void MySQLClient::registerSlaveOnMaster(UInt32 slave_id)
|
||||
packet_endpoint->receivePacket(packet_response);
|
||||
packet_endpoint->resetSequenceId();
|
||||
if (packet_response.getType() == PACKET_ERR)
|
||||
throw Exception(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
|
||||
throw Exception::createDeprecated(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
|
||||
}
|
||||
|
||||
void MySQLClient::ping()
|
||||
|
||||
@ -111,7 +111,7 @@ namespace MySQLReplication
|
||||
else if (query.starts_with("XA"))
|
||||
{
|
||||
if (query.starts_with("XA ROLLBACK"))
|
||||
throw ReplicationError("ParseQueryEvent: Unsupported query event:" + query, ErrorCodes::LOGICAL_ERROR);
|
||||
throw ReplicationError(ErrorCodes::LOGICAL_ERROR, "ParseQueryEvent: Unsupported query event: {}", query);
|
||||
typ = QUERY_EVENT_XA;
|
||||
if (!query.starts_with("XA COMMIT"))
|
||||
transaction_complete = false;
|
||||
@ -247,7 +247,7 @@ namespace MySQLReplication
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw ReplicationError("ParseMetaData: Unhandled data type:" + std::to_string(typ), ErrorCodes::UNKNOWN_EXCEPTION);
|
||||
throw ReplicationError(ErrorCodes::UNKNOWN_EXCEPTION, "ParseMetaData: Unhandled data type: {}", std::to_string(typ));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -770,8 +770,8 @@ namespace MySQLReplication
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw ReplicationError(
|
||||
"ParseRow: Unhandled MySQL field type:" + std::to_string(field_type), ErrorCodes::UNKNOWN_EXCEPTION);
|
||||
throw ReplicationError(ErrorCodes::UNKNOWN_EXCEPTION,
|
||||
"ParseRow: Unhandled MySQL field type: {}", std::to_string(field_type));
|
||||
}
|
||||
}
|
||||
null_index++;
|
||||
@ -873,7 +873,7 @@ namespace MySQLReplication
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw ReplicationError("Position update with unsupported event", ErrorCodes::LOGICAL_ERROR);
|
||||
throw ReplicationError(ErrorCodes::LOGICAL_ERROR, "Position update with unsupported event");
|
||||
}
|
||||
}
|
||||
|
||||
@ -901,11 +901,11 @@ namespace MySQLReplication
|
||||
switch (header)
|
||||
{
|
||||
case PACKET_EOF:
|
||||
throw ReplicationError("Master maybe lost", ErrorCodes::CANNOT_READ_ALL_DATA);
|
||||
throw ReplicationError(ErrorCodes::CANNOT_READ_ALL_DATA, "Master maybe lost");
|
||||
case PACKET_ERR:
|
||||
ERRPacket err;
|
||||
err.readPayloadWithUnpacked(payload);
|
||||
throw ReplicationError(err.error_message, ErrorCodes::UNKNOWN_EXCEPTION);
|
||||
throw ReplicationError::createDeprecated(err.error_message, ErrorCodes::UNKNOWN_EXCEPTION);
|
||||
}
|
||||
// skip the generic response packets header flag.
|
||||
payload.ignore(1);
|
||||
|
||||
@ -74,7 +74,7 @@ ConnectionHolderPtr PoolWithFailover::get()
|
||||
if (replicas_with_priority.empty())
|
||||
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "No address specified");
|
||||
|
||||
DB::WriteBufferFromOwnString error_message;
|
||||
PreformattedMessage error_message;
|
||||
for (size_t try_idx = 0; try_idx < max_tries; ++try_idx)
|
||||
{
|
||||
for (auto & priority : replicas_with_priority)
|
||||
@ -107,7 +107,7 @@ ConnectionHolderPtr PoolWithFailover::get()
|
||||
catch (const pqxx::broken_connection & pqxx_error)
|
||||
{
|
||||
LOG_ERROR(log, "Connection error: {}", pqxx_error.what());
|
||||
error_message << fmt::format(
|
||||
error_message = PreformattedMessage::create(
|
||||
"Try {}. Connection to {} failed with error: {}\n",
|
||||
try_idx + 1, DB::backQuote(replica.connection_info.host_port), pqxx_error.what());
|
||||
|
||||
@ -131,7 +131,7 @@ ConnectionHolderPtr PoolWithFailover::get()
|
||||
}
|
||||
}
|
||||
|
||||
throw DB::Exception(DB::ErrorCodes::POSTGRESQL_CONNECTION_FAILURE, error_message.str());
|
||||
throw DB::Exception(error_message, DB::ErrorCodes::POSTGRESQL_CONNECTION_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -584,7 +584,7 @@ class IColumn;
|
||||
M(Bool, query_plan_optimize_primary_key, true, "Analyze primary key using query plan (instead of AST)", 0) \
|
||||
M(Bool, query_plan_read_in_order, true, "Use query plan for read-in-order optimisation", 0) \
|
||||
M(Bool, query_plan_aggregation_in_order, true, "Use query plan for aggregation-in-order optimisation", 0) \
|
||||
M(Bool, query_plan_remove_redundant_sorting, false, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries", 0) \
|
||||
M(Bool, query_plan_remove_redundant_sorting, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries", 0) \
|
||||
M(UInt64, regexp_max_matches_per_row, 1000, "Max matches of any single regexp per row, used to safeguard 'extractAllGroupsHorizontal' against consuming too much memory with greedy RE.", 0) \
|
||||
\
|
||||
M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \
|
||||
@ -784,6 +784,7 @@ class IColumn;
|
||||
M(Bool, input_format_json_read_numbers_as_strings, false, "Allow to parse numbers as strings in JSON input formats", 0) \
|
||||
M(Bool, input_format_json_read_objects_as_strings, true, "Allow to parse JSON objects as strings in JSON input formats", 0) \
|
||||
M(Bool, input_format_json_named_tuples_as_objects, true, "Deserialize named tuple columns as JSON objects", 0) \
|
||||
M(Bool, input_format_json_ignore_unknown_keys_in_named_tuple, false, "Ignore unknown keys in json object for named tuples", 0) \
|
||||
M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \
|
||||
M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
|
||||
M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
|
||||
|
||||
@ -81,10 +81,11 @@ namespace SettingsChangesHistory
|
||||
static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> settings_changes_history =
|
||||
{
|
||||
{"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
|
||||
{"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
|
||||
{"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
|
||||
{"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
|
||||
{"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
|
||||
{"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}}},
|
||||
{"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}},
|
||||
{"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
|
||||
{"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
|
||||
{"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
|
||||
|
||||
@ -402,7 +402,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
|
||||
auto it = map.find(value); \
|
||||
if (it != map.end()) \
|
||||
return it->second; \
|
||||
throw Exception( \
|
||||
throw Exception::createDeprecated( \
|
||||
"Unexpected value of " #NEW_NAME ":" + std::to_string(std::underlying_type<EnumType>::type(value)), \
|
||||
ERROR_CODE_FOR_UNEXPECTED_NAME); \
|
||||
} \
|
||||
@ -428,7 +428,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
|
||||
msg += "'" + String{name} + "'"; \
|
||||
} \
|
||||
msg += "]"; \
|
||||
throw Exception(msg, ERROR_CODE_FOR_UNEXPECTED_NAME); \
|
||||
throw Exception::createDeprecated(msg, ERROR_CODE_FOR_UNEXPECTED_NAME); \
|
||||
}
|
||||
|
||||
// Mostly like SettingFieldEnum, but can have multiple enum values (or none) set at once.
|
||||
|
||||
@ -933,7 +933,7 @@ void BaseDaemon::handleSignal(int signal_id)
|
||||
onInterruptSignals(signal_id);
|
||||
}
|
||||
else
|
||||
throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
|
||||
throw DB::Exception::createDeprecated(std::string("Unsupported signal: ") + strsignal(signal_id), 0); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
|
||||
}
|
||||
|
||||
void BaseDaemon::onInterruptSignals(int signal_id)
|
||||
|
||||
@ -189,7 +189,8 @@ template <template <typename> typename DecimalType>
|
||||
inline DataTypePtr createDecimal(UInt64 precision_value, UInt64 scale_value)
|
||||
{
|
||||
if (precision_value < DecimalUtils::min_precision || precision_value > DecimalUtils::max_precision<Decimal256>)
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Wrong precision");
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Wrong precision: it must be between {} and {}, got {}",
|
||||
DecimalUtils::min_precision, DecimalUtils::max_precision<Decimal256>, precision_value);
|
||||
|
||||
if (static_cast<UInt64>(scale_value) > precision_value)
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Negative scales and scales larger than precision are not supported");
|
||||
|
||||
@ -57,7 +57,7 @@ static DataTypePtr create(const ASTPtr & arguments)
|
||||
{
|
||||
if (func->name != "Nullable" || func->arguments->children.size() != 1)
|
||||
throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE,
|
||||
"Expected 'Nullable(<schema_name>)' as parameter for type Object", func->name);
|
||||
"Expected 'Nullable(<schema_name>)' as parameter for type Object (function: {})", func->name);
|
||||
|
||||
schema_argument = func->arguments->children[0];
|
||||
is_nullable = true;
|
||||
|
||||
@ -53,10 +53,10 @@ static std::optional<Exception> checkTupleNames(const Strings & names)
|
||||
for (const auto & name : names)
|
||||
{
|
||||
if (name.empty())
|
||||
return Exception("Names of tuple elements cannot be empty", ErrorCodes::BAD_ARGUMENTS);
|
||||
return Exception(ErrorCodes::BAD_ARGUMENTS, "Names of tuple elements cannot be empty");
|
||||
|
||||
if (!names_set.insert(name).second)
|
||||
return Exception("Names of tuple elements must be unique", ErrorCodes::DUPLICATE_COLUMN);
|
||||
return Exception(ErrorCodes::DUPLICATE_COLUMN, "Names of tuple elements must be unique");
|
||||
}
|
||||
|
||||
return {};
|
||||
|
||||
@ -373,8 +373,8 @@ void SerializationArray::deserializeBinaryBulkWithMultipleStreams(
|
||||
/// Check consistency between offsets and elements subcolumns.
|
||||
/// But if elements column is empty - it's ok for columns of Nested types that was added by ALTER.
|
||||
if (!nested_column->empty() && nested_column->size() != last_offset)
|
||||
throw ParsingException("Cannot read all array values: read just " + toString(nested_column->size()) + " of " + toString(last_offset),
|
||||
ErrorCodes::CANNOT_READ_ALL_DATA);
|
||||
throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read all array values: read just {} of {}",
|
||||
toString(nested_column->size()), toString(last_offset));
|
||||
|
||||
column = std::move(mutable_column);
|
||||
}
|
||||
|
||||
@ -360,19 +360,20 @@ ReturnType SerializationNullable::deserializeTextEscapedAndRawImpl(IColumn & col
|
||||
/// or if someone uses tab or LF in TSV null_representation.
|
||||
/// In the first case we cannot continue reading anyway. The second case seems to be unlikely.
|
||||
if (null_representation.find('\t') != std::string::npos || null_representation.find('\n') != std::string::npos)
|
||||
throw DB::ParsingException("TSV custom null representation containing '\\t' or '\\n' may not work correctly "
|
||||
"for large input.", ErrorCodes::CANNOT_READ_ALL_DATA);
|
||||
throw DB::ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "TSV custom null representation "
|
||||
"containing '\\t' or '\\n' may not work correctly for large input.");
|
||||
|
||||
WriteBufferFromOwnString parsed_value;
|
||||
if constexpr (escaped)
|
||||
nested_serialization->serializeTextEscaped(nested_column, nested_column.size() - 1, parsed_value, settings);
|
||||
else
|
||||
nested_serialization->serializeTextRaw(nested_column, nested_column.size() - 1, parsed_value, settings);
|
||||
throw DB::ParsingException("Error while parsing \"" + std::string(pos, buf.buffer().end()) + std::string(istr.position(), std::min(size_t(10), istr.available())) + "\" as Nullable"
|
||||
+ " at position " + std::to_string(istr.count()) + ": got \"" + std::string(pos, buf.position() - pos)
|
||||
+ "\", which was deserialized as \""
|
||||
+ parsed_value.str() + "\". It seems that input data is ill-formatted.",
|
||||
ErrorCodes::CANNOT_READ_ALL_DATA);
|
||||
throw DB::ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Error while parsing \"{}{}\" as Nullable"
|
||||
" at position {}: got \"{}\", which was deserialized as \"{}\". "
|
||||
"It seems that input data is ill-formatted.",
|
||||
std::string(pos, buf.buffer().end()),
|
||||
std::string(istr.position(), std::min(size_t(10), istr.available())),
|
||||
istr.count(), std::string(pos, buf.position() - pos), parsed_value.str());
|
||||
};
|
||||
|
||||
return safeDeserialize<ReturnType>(column, *nested_serialization, check_for_null, deserialize_nested);
|
||||
@ -584,16 +585,17 @@ ReturnType SerializationNullable::deserializeTextCSVImpl(IColumn & column, ReadB
|
||||
/// In the first case we cannot continue reading anyway. The second case seems to be unlikely.
|
||||
if (null_representation.find(settings.csv.delimiter) != std::string::npos || null_representation.find('\r') != std::string::npos
|
||||
|| null_representation.find('\n') != std::string::npos)
|
||||
throw DB::ParsingException("CSV custom null representation containing format_csv_delimiter, '\\r' or '\\n' may not work correctly "
|
||||
"for large input.", ErrorCodes::CANNOT_READ_ALL_DATA);
|
||||
throw DB::ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "CSV custom null representation containing "
|
||||
"format_csv_delimiter, '\\r' or '\\n' may not work correctly for large input.");
|
||||
|
||||
WriteBufferFromOwnString parsed_value;
|
||||
nested_serialization->serializeTextCSV(nested_column, nested_column.size() - 1, parsed_value, settings);
|
||||
throw DB::ParsingException("Error while parsing \"" + std::string(pos, buf.buffer().end()) + std::string(istr.position(), std::min(size_t(10), istr.available())) + "\" as Nullable"
|
||||
+ " at position " + std::to_string(istr.count()) + ": got \"" + std::string(pos, buf.position() - pos)
|
||||
+ "\", which was deserialized as \""
|
||||
+ parsed_value.str() + "\". It seems that input data is ill-formatted.",
|
||||
ErrorCodes::CANNOT_READ_ALL_DATA);
|
||||
throw DB::ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Error while parsing \"{}{}\" as Nullable"
|
||||
" at position {}: got \"{}\", which was deserialized as \"{}\". "
|
||||
"It seems that input data is ill-formatted.",
|
||||
std::string(pos, buf.buffer().end()),
|
||||
std::string(istr.position(), std::min(size_t(10), istr.available())),
|
||||
istr.count(), std::string(pos, buf.position() - pos), parsed_value.str());
|
||||
};
|
||||
|
||||
return safeDeserialize<ReturnType>(column, *nested_serialization, check_for_null, deserialize_nested);
|
||||
|
||||
@ -231,7 +231,7 @@ void SerializationObject<Parser>::deserializeBinaryBulkStatePrefix(
|
||||
auto kind = magic_enum::enum_cast<BinarySerializationKind>(kind_raw);
|
||||
if (!kind)
|
||||
throw Exception(ErrorCodes::INCORRECT_DATA,
|
||||
"Unknown binary serialization kind of Object: " + std::to_string(kind_raw));
|
||||
"Unknown binary serialization kind of Object: {}", std::to_string(kind_raw));
|
||||
|
||||
auto state_object = std::make_shared<DeserializeStateObject>();
|
||||
state_object->kind = *kind;
|
||||
@ -255,7 +255,7 @@ void SerializationObject<Parser>::deserializeBinaryBulkStatePrefix(
|
||||
else
|
||||
{
|
||||
throw Exception(ErrorCodes::INCORRECT_DATA,
|
||||
"Unknown binary serialization kind of Object: " + std::to_string(kind_raw));
|
||||
"Unknown binary serialization kind of Object: {}", std::to_string(kind_raw));
|
||||
}
|
||||
|
||||
settings.path.push_back(Substream::ObjectData);
|
||||
|
||||
@ -196,13 +196,14 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr
|
||||
addElementSafe(elems.size(), column, [&]
|
||||
{
|
||||
std::vector<UInt8> seen_elements(elems.size(), 0);
|
||||
size_t i = 0;
|
||||
size_t processed = 0;
|
||||
size_t skipped = 0;
|
||||
while (!istr.eof() && *istr.position() != '}')
|
||||
{
|
||||
if (i == elems.size())
|
||||
throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected number of elements in named tuple. Expected no more than {}", elems.size());
|
||||
if (!settings.json.ignore_unknown_keys_in_named_tuple && processed == elems.size())
|
||||
throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected number of elements in named tuple. Expected no more than {} (consider enabling input_format_json_ignore_unknown_keys_in_named_tuple setting)", elems.size());
|
||||
|
||||
if (i > 0)
|
||||
if (processed + skipped > 0)
|
||||
{
|
||||
assertChar(',', istr);
|
||||
skipWhitespaceIfAny(istr);
|
||||
@ -215,18 +216,31 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr
|
||||
skipWhitespaceIfAny(istr);
|
||||
|
||||
const size_t element_pos = getPositionByName(name);
|
||||
if (element_pos == std::numeric_limits<size_t>::max())
|
||||
{
|
||||
if (settings.json.ignore_unknown_keys_in_named_tuple)
|
||||
{
|
||||
skipJSONField(istr, name);
|
||||
skipWhitespaceIfAny(istr);
|
||||
++skipped;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK, "Tuple doesn't have element with name '{}', enable setting input_format_json_ignore_unknown_keys_in_named_tuple", name);
|
||||
}
|
||||
|
||||
seen_elements[element_pos] = 1;
|
||||
auto & element_column = extractElementColumn(column, element_pos);
|
||||
elems[element_pos]->deserializeTextJSON(element_column, istr, settings);
|
||||
|
||||
skipWhitespaceIfAny(istr);
|
||||
++i;
|
||||
++processed;
|
||||
}
|
||||
|
||||
assertChar('}', istr);
|
||||
|
||||
/// Check if we have missing elements.
|
||||
if (i != elems.size())
|
||||
if (processed != elems.size())
|
||||
{
|
||||
for (size_t element_pos = 0; element_pos != seen_elements.size(); ++element_pos)
|
||||
{
|
||||
@ -416,7 +430,7 @@ size_t SerializationTuple::getPositionByName(const String & name) const
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
if (elems[i]->getElementName() == name)
|
||||
return i;
|
||||
throw Exception(ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK, "Tuple doesn't have element with name '{}'", name);
|
||||
return std::numeric_limits<size_t>::max();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user