mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge branch 'master' into add_separate_pool_for_fetches
This commit is contained in:
commit
2e357516a6
@ -17,13 +17,6 @@ def get_skip_list_cmd(path):
|
||||
return ''
|
||||
|
||||
|
||||
def run_perf_test(cmd, xmls_path, output_folder):
|
||||
output_path = os.path.join(output_folder, "perf_stress_run.txt")
|
||||
f = open(output_path, 'w')
|
||||
p = Popen("{} --skip-tags=long --recursive --input-files {}".format(cmd, xmls_path), shell=True, stdout=f, stderr=f)
|
||||
return p
|
||||
|
||||
|
||||
def get_options(i):
|
||||
options = ""
|
||||
if 0 < i:
|
||||
@ -75,8 +68,6 @@ if __name__ == "__main__":
|
||||
|
||||
args = parser.parse_args()
|
||||
func_pipes = []
|
||||
perf_process = None
|
||||
perf_process = run_perf_test(args.perf_test_cmd, args.perf_test_xml_path, args.output_folder)
|
||||
func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests, args.global_time_limit)
|
||||
|
||||
logging.info("Will wait functests to finish")
|
||||
|
141
docs/en/development/adding_test_queries.md
Normal file
141
docs/en/development/adding_test_queries.md
Normal file
@ -0,0 +1,141 @@
|
||||
# How to add test queries to ClickHouse CI
|
||||
|
||||
ClickHouse has hundreds (or even thousands) of features. Every commit get checked by a complex set of tests containing many thousands of test cases.
|
||||
|
||||
The core functionality is very well tested, but some corner-cases and different combinations of features can be uncovered with ClickHouse CI.
|
||||
|
||||
Most of the bugs/regressions we see happen in that 'grey area' where test coverage is poor.
|
||||
|
||||
And we are very interested in covering most of the possible scenarios and feature combinations used in real life by tests.
|
||||
|
||||
## Why adding tests
|
||||
|
||||
Why/when you should add a test case into ClickHouse code:
|
||||
1) you use some complicated scenarios / feature combinations / you have some corner case which is probably not widely used
|
||||
2) you see that certain behavior gets changed between version w/o notifications in the changelog
|
||||
3) you just want to help to improve ClickHouse quality and ensure the features you use will not be broken in the future releases
|
||||
4) once the test is added/accepted, you can be sure the corner case you check will never be accidentally broken.
|
||||
5) you will be a part of great open-source community
|
||||
6) your name will be visible in the `system.contributors` table!
|
||||
7) you will make a world bit better :)
|
||||
|
||||
### Steps to do
|
||||
|
||||
#### Prerequisite
|
||||
|
||||
I assume you run some Linux machine (you can use docker / virtual machines on other OS) and any modern browser / internet connection, and you have some basic Linux & SQL skills.
|
||||
|
||||
Any highly specialized knowledge is not needed (so you don't need to know C++ or know something about how ClickHouse CI works).
|
||||
|
||||
|
||||
#### Preparation
|
||||
|
||||
1) [create GitHub account](https://github.com/join) (if you haven't one yet)
|
||||
2) [setup git](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/set-up-git)
|
||||
```bash
|
||||
# for Ubuntu
|
||||
sudo apt-get update
|
||||
sudo apt-get install git
|
||||
|
||||
git config --global user.name "John Doe" # fill with your name
|
||||
git config --global user.email "email@example.com" # fill with your email
|
||||
|
||||
```
|
||||
3) [fork ClickHouse project](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/fork-a-repo) - just open [https://github.com/ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse) and press fork button in the top right corner:
|
||||
![fork repo](https://github-images.s3.amazonaws.com/help/bootcamp/Bootcamp-Fork.png)
|
||||
|
||||
4) clone your fork to some folder on your PC, for example, `~/workspace/ClickHouse`
|
||||
```
|
||||
mkdir ~/workspace && cd ~/workspace
|
||||
git clone https://github.com/< your GitHub username>/ClickHouse
|
||||
cd ClickHouse
|
||||
git remote add upstream https://github.com/ClickHouse/ClickHouse
|
||||
```
|
||||
|
||||
#### New branch for the test
|
||||
|
||||
1) create a new branch from the latest clickhouse master
|
||||
```
|
||||
cd ~/workspace/ClickHouse
|
||||
git fetch upstream
|
||||
git checkout -b name_for_a_branch_with_my_test upstream/master
|
||||
```
|
||||
|
||||
#### Install & run clickhouse
|
||||
|
||||
1) install `clickhouse-server` (follow [official docs](https://clickhouse.tech/docs/en/getting-started/install/))
|
||||
2) install test configurations (it will use Zookeeper mock implementation and adjust some settings)
|
||||
```
|
||||
cd ~/workspace/ClickHouse/tests/config
|
||||
sudo ./install.sh
|
||||
```
|
||||
3) run clickhouse-server
|
||||
```
|
||||
sudo systemctl restart clickhouse-server
|
||||
```
|
||||
|
||||
#### Creating the test file
|
||||
|
||||
|
||||
1) find the number for your test - find the file with the biggest number in `tests/queries/0_stateless/`
|
||||
|
||||
```sh
|
||||
$ cd ~/workspace/ClickHouse
|
||||
$ ls tests/queries/0_stateless/[0-9]*.reference | tail -n 1
|
||||
tests/queries/0_stateless/01520_client_print_query_id.reference
|
||||
```
|
||||
Currently, the last number for the test is `01520`, so my test will have the number `01521`
|
||||
|
||||
2) create an SQL file with the next number and name of the feature you test
|
||||
|
||||
```sh
|
||||
touch tests/queries/0_stateless/01521_dummy_test.sql
|
||||
```
|
||||
|
||||
3) edit SQL file with your favorite editor (see hint of creating tests below)
|
||||
```sh
|
||||
vim tests/queries/0_stateless/01521_dummy_test.sql
|
||||
```
|
||||
|
||||
|
||||
4) run the test, and put the result of that into the reference file:
|
||||
```
|
||||
clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee tests/queries/0_stateless/01521_dummy_test.reference
|
||||
```
|
||||
|
||||
5) ensure everything is correct, if the test output is incorrect (due to some bug for example), adjust the reference file using text editor.
|
||||
|
||||
#### How create good test
|
||||
|
||||
- test should be
|
||||
- minimal - create only tables related to tested functionality, remove unrelated columns and parts of query
|
||||
- fast - should not take longer than few seconds (better subseconds)
|
||||
- correct - fails then feature is not working
|
||||
- deteministic
|
||||
- isolated / stateless
|
||||
- don't rely on some environment things
|
||||
- don't rely on timing when possible
|
||||
- try to cover corner cases (zeros / Nulls / empty sets / throwing exceptions)
|
||||
- to test that query return errors, you can put special comment after the query: `-- { serverError 60 }` or `-- { clientError 20 }`
|
||||
- don't switch databases (unless necessary)
|
||||
- you can create several table replicas on the same node if needed
|
||||
- you can use one of the test cluster definitions when needed (see system.clusters)
|
||||
- use `number` / `numbers_mt` / `zeros` / `zeros_mt` and similar for queries / to initialize data when appliable
|
||||
- clean up the created objects after test and before the test (DROP IF EXISTS) - in case of some dirty state
|
||||
- prefer sync mode of operations (mutations, merges, etc.)
|
||||
- use other SQL files in the `0_stateless` folder as an example
|
||||
- ensure the feature / feature combination you want to tests is not covered yet with existsing tests
|
||||
|
||||
#### Commit / push / create PR.
|
||||
|
||||
1) commit & push your changes
|
||||
```sh
|
||||
cd ~/workspace/ClickHouse
|
||||
git add tests/queries/0_stateless/01521_dummy_test.sql
|
||||
git add tests/queries/0_stateless/01521_dummy_test.reference
|
||||
git commit # use some nice commit message when possible
|
||||
git push origin HEAD
|
||||
```
|
||||
2) use a link which was shown during the push, to create a PR into the main repo
|
||||
3) adjust the PR title and contents, in `Changelog category (leave one)` keep
|
||||
`Build/Testing/Packaging Improvement`, fill the rest of the fields if you want.
|
@ -384,7 +384,7 @@ Possible values:
|
||||
|
||||
- `'basic'` — Use basic parser.
|
||||
|
||||
ClickHouse can parse only the basic `YYYY-MM-DD HH:MM:SS` format. For example, `'2019-08-20 10:18:56'`.
|
||||
ClickHouse can parse only the basic `YYYY-MM-DD HH:MM:SS` or `YYYY-MM-DD` format. For example, `'2019-08-20 10:18:56'` or `2019-08-20`.
|
||||
|
||||
Default value: `'basic'`.
|
||||
|
||||
|
@ -3,10 +3,45 @@ toc_priority: 47
|
||||
toc_title: Date
|
||||
---
|
||||
|
||||
# Date {#date}
|
||||
# Date {#data_type-date}
|
||||
|
||||
A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2106, but the final fully-supported year is 2105).
|
||||
|
||||
The date value is stored without the time zone.
|
||||
|
||||
## Examples {#examples}
|
||||
|
||||
**1.** Creating a table with a `DateTime`-type column and inserting data into it:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE dt
|
||||
(
|
||||
`timestamp` Date,
|
||||
`event_id` UInt8
|
||||
)
|
||||
ENGINE = TinyLog;
|
||||
```
|
||||
|
||||
``` sql
|
||||
INSERT INTO dt Values (1546300800, 1), ('2019-01-01', 2);
|
||||
```
|
||||
|
||||
``` sql
|
||||
SELECT * FROM dt;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌──timestamp─┬─event_id─┐
|
||||
│ 2019-01-01 │ 1 │
|
||||
│ 2019-01-01 │ 2 │
|
||||
└────────────┴──────────┘
|
||||
```
|
||||
|
||||
## See Also {#see-also}
|
||||
|
||||
- [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md)
|
||||
- [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime)
|
||||
- [`DateTime` data type](../../sql-reference/data-types/datetime.md)
|
||||
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/data_types/date/) <!--hide-->
|
||||
|
@ -221,3 +221,85 @@ returns
|
||||
│ 1970-03-12 │ 1970-01-08 │ original │
|
||||
└────────────┴────────────┴──────────┘
|
||||
```
|
||||
|
||||
## OFFSET FETCH Clause {#offset-fetch}
|
||||
|
||||
`OFFSET` and `FETCH` allow you to retrieve data by portions. They specify a row block which you want to get by a single query.
|
||||
|
||||
``` sql
|
||||
OFFSET offset_row_count {ROW | ROWS}] [FETCH {FIRST | NEXT} fetch_row_count {ROW | ROWS} {ONLY | WITH TIES}]
|
||||
```
|
||||
|
||||
The `offset_row_count` or `fetch_row_count` value can be a number or a literal constant. You can omit `fetch_row_count`; by default, it equals 1.
|
||||
|
||||
`OFFSET` specifies the number of rows to skip before starting to return rows from the query.
|
||||
|
||||
The `FETCH` specifies the maximum number of rows that can be in the result of a query.
|
||||
|
||||
The `ONLY` option is used to return rows that immediately follow the rows omitted by the `OFFSET`. In this case the `FETCH` is an alternative to the [LIMIT](../../../sql-reference/statements/select/limit.md) clause. For example, the following query
|
||||
|
||||
``` sql
|
||||
SELECT * FROM test_fetch ORDER BY a OFFSET 1 ROW FETCH FIRST 3 ROWS ONLY;
|
||||
```
|
||||
|
||||
is identical to the query
|
||||
|
||||
``` sql
|
||||
SELECT * FROM test_fetch ORDER BY a LIMIT 3 OFFSET 1;
|
||||
```
|
||||
|
||||
The `WITH TIES` option is used to return any additional rows that tie for the last place in the result set according to the `ORDER BY` clause. For example, if `fetch_row_count` is set to 5 but two additional rows match the values of the `ORDER BY` columns in the fifth row, the result set will contain seven rows.
|
||||
|
||||
!!! note "Note"
|
||||
According to the standard, the `OFFSET` clause must come before the `FETCH` clause if both are present.
|
||||
|
||||
### Examples {#examples}
|
||||
|
||||
Input table:
|
||||
|
||||
``` text
|
||||
┌─a─┬─b─┐
|
||||
│ 1 │ 1 │
|
||||
│ 2 │ 1 │
|
||||
│ 3 │ 4 │
|
||||
│ 1 │ 3 │
|
||||
│ 5 │ 4 │
|
||||
│ 0 │ 6 │
|
||||
│ 5 │ 7 │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
Usage of the `ONLY` option:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM test_fetch ORDER BY a OFFSET 3 ROW FETCH FIRST 3 ROWS ONLY;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─a─┬─b─┐
|
||||
│ 2 │ 1 │
|
||||
│ 3 │ 4 │
|
||||
│ 5 │ 4 │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
Usage of the `WITH TIES` option:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM test_fetch ORDER BY a OFFSET 3 ROW FETCH FIRST 3 ROWS WITH TIES;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─a─┬─b─┐
|
||||
│ 2 │ 1 │
|
||||
│ 3 │ 4 │
|
||||
│ 5 │ 4 │
|
||||
│ 5 │ 7 │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/order-by/) <!--hide-->
|
||||
|
@ -214,3 +214,85 @@ ORDER BY
|
||||
│ 1970-03-12 │ 1970-01-08 │ original │
|
||||
└────────────┴────────────┴──────────┘
|
||||
```
|
||||
|
||||
## Секция OFFSET FETCH {#offset-fetch}
|
||||
|
||||
`OFFSET` и `FETCH` позволяют извлекать данные по частям. Они указывают строки, которые вы хотите получить в результате запроса.
|
||||
|
||||
``` sql
|
||||
OFFSET offset_row_count {ROW | ROWS}] [FETCH {FIRST | NEXT} fetch_row_count {ROW | ROWS} {ONLY | WITH TIES}]
|
||||
```
|
||||
|
||||
`offset_row_count` или `fetch_row_count` может быть числом или литеральной константой. Если вы не используете `fetch_row_count`, то его значение равно 1.
|
||||
|
||||
`OFFSET` указывает количество строк, которые необходимо пропустить перед началом возврата строк из запроса.
|
||||
|
||||
`FETCH` указывает максимальное количество строк, которые могут быть получены в результате запроса.
|
||||
|
||||
Опция `ONLY` используется для возврата строк, которые следуют сразу же за строками, пропущенными секцией `OFFSET`. В этом случае `FETCH` — это альтернатива [LIMIT](../../../sql-reference/statements/select/limit.md). Например, следующий запрос
|
||||
|
||||
``` sql
|
||||
SELECT * FROM test_fetch ORDER BY a OFFSET 1 ROW FETCH FIRST 3 ROWS ONLY;
|
||||
```
|
||||
|
||||
идентичен запросу
|
||||
|
||||
``` sql
|
||||
SELECT * FROM test_fetch ORDER BY a LIMIT 3 OFFSET 1;
|
||||
```
|
||||
|
||||
Опция `WITH TIES` используется для возврата дополнительных строк, которые привязываются к последней в результате запроса. Например, если `fetch_row_count` имеет значение 5 и существуют еще 2 строки с такими же значениями столбцов, указанных в `ORDER BY`, что и у пятой строки результата, то финальный набор будет содержать 7 строк.
|
||||
|
||||
!!! note "Примечание"
|
||||
Секция `OFFSET` должна находиться перед секцией `FETCH`, если обе присутствуют.
|
||||
|
||||
### Примеры {#examples}
|
||||
|
||||
Входная таблица:
|
||||
|
||||
``` text
|
||||
┌─a─┬─b─┐
|
||||
│ 1 │ 1 │
|
||||
│ 2 │ 1 │
|
||||
│ 3 │ 4 │
|
||||
│ 1 │ 3 │
|
||||
│ 5 │ 4 │
|
||||
│ 0 │ 6 │
|
||||
│ 5 │ 7 │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
Использование опции `ONLY`:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM test_fetch ORDER BY a OFFSET 3 ROW FETCH FIRST 3 ROWS ONLY;
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─a─┬─b─┐
|
||||
│ 2 │ 1 │
|
||||
│ 3 │ 4 │
|
||||
│ 5 │ 4 │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
Использование опции `WITH TIES`:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM test_fetch ORDER BY a OFFSET 3 ROW FETCH FIRST 3 ROWS WITH TIES;
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─a─┬─b─┐
|
||||
│ 2 │ 1 │
|
||||
│ 3 │ 4 │
|
||||
│ 5 │ 4 │
|
||||
│ 5 │ 7 │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/order-by/) <!--hide-->
|
||||
|
@ -130,4 +130,6 @@ void Settings::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfigura
|
||||
}
|
||||
}
|
||||
|
||||
IMPLEMENT_SETTINGS_TRAITS(FormatFactorySettingsTraits, FORMAT_FACTORY_SETTINGS)
|
||||
|
||||
}
|
||||
|
@ -515,4 +515,13 @@ struct Settings : public BaseSettings<SettingsTraits>
|
||||
static void checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path);
|
||||
};
|
||||
|
||||
/*
|
||||
* User-specified file format settings for File and ULR engines.
|
||||
*/
|
||||
DECLARE_SETTINGS_TRAITS(FormatFactorySettingsTraits, FORMAT_FACTORY_SETTINGS)
|
||||
|
||||
struct FormatFactorySettings : public BaseSettings<FormatFactorySettingsTraits>
|
||||
{
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -623,24 +623,10 @@ void MaterializeMySQLSyncThread::onEvent(Buffers & buffers, const BinlogEventPtr
|
||||
else if (receive_event->type() == MYSQL_QUERY_EVENT)
|
||||
{
|
||||
QueryEvent & query_event = static_cast<QueryEvent &>(*receive_event);
|
||||
flushBuffersData(buffers, metadata);
|
||||
|
||||
try
|
||||
{
|
||||
Context query_context = createQueryContext(global_context);
|
||||
String comment = "Materialize MySQL step 2: execute MySQL DDL for sync data";
|
||||
String event_database = query_event.schema == mysql_database_name ? database_name : "";
|
||||
tryToExecuteQuery(query_prefix + query_event.query, query_context, event_database, comment);
|
||||
}
|
||||
catch (Exception & exception)
|
||||
{
|
||||
tryLogCurrentException(log);
|
||||
|
||||
/// If some DDL query was not successfully parsed and executed
|
||||
/// Then replication may fail on next binlog events anyway
|
||||
if (exception.code() != ErrorCodes::SYNTAX_ERROR)
|
||||
throw;
|
||||
}
|
||||
Position position_before_ddl;
|
||||
position_before_ddl.update(metadata.binlog_position, metadata.binlog_file, metadata.executed_gtid_set);
|
||||
metadata.transaction(position_before_ddl, [&]() { buffers.commit(global_context); });
|
||||
metadata.transaction(client.getPosition(),[&](){ executeDDLAtomic(query_event); });
|
||||
}
|
||||
else if (receive_event->header.type != HEARTBEAT_EVENT)
|
||||
{
|
||||
@ -656,6 +642,26 @@ void MaterializeMySQLSyncThread::onEvent(Buffers & buffers, const BinlogEventPtr
|
||||
}
|
||||
}
|
||||
|
||||
void MaterializeMySQLSyncThread::executeDDLAtomic(const QueryEvent & query_event)
|
||||
{
|
||||
try
|
||||
{
|
||||
Context query_context = createQueryContext(global_context);
|
||||
String comment = "Materialize MySQL step 2: execute MySQL DDL for sync data";
|
||||
String event_database = query_event.schema == mysql_database_name ? database_name : "";
|
||||
tryToExecuteQuery(query_prefix + query_event.query, query_context, event_database, comment);
|
||||
}
|
||||
catch (Exception & exception)
|
||||
{
|
||||
tryLogCurrentException(log);
|
||||
|
||||
/// If some DDL query was not successfully parsed and executed
|
||||
/// Then replication may fail on next binlog events anyway
|
||||
if (exception.code() != ErrorCodes::SYNTAX_ERROR)
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
bool MaterializeMySQLSyncThread::isMySQLSyncThread()
|
||||
{
|
||||
return getThreadName() == MYSQL_BACKGROUND_THREAD_NAME;
|
||||
|
@ -100,6 +100,7 @@ private:
|
||||
|
||||
std::atomic<bool> sync_quit{false};
|
||||
std::unique_ptr<ThreadFromGlobalPool> background_thread_pool;
|
||||
void executeDDLAtomic(const QueryEvent & query_event);
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -40,100 +40,93 @@ const FormatFactory::Creators & FormatFactory::getCreators(const String & name)
|
||||
throw Exception("Unknown format " + name, ErrorCodes::UNKNOWN_FORMAT);
|
||||
}
|
||||
|
||||
FormatSettings getFormatSettings(const Context & context)
|
||||
{
|
||||
const auto & settings = context.getSettingsRef();
|
||||
|
||||
static FormatSettings getInputFormatSetting(const Settings & settings, const Context & context)
|
||||
return getFormatSettings(context, settings);
|
||||
}
|
||||
|
||||
template <typename Settings>
|
||||
FormatSettings getFormatSettings(const Context & context,
|
||||
const Settings & settings)
|
||||
{
|
||||
FormatSettings format_settings;
|
||||
format_settings.csv.delimiter = settings.format_csv_delimiter;
|
||||
format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes;
|
||||
|
||||
format_settings.avro.allow_missing_fields = settings.input_format_avro_allow_missing_fields;
|
||||
format_settings.avro.output_codec = settings.output_format_avro_codec;
|
||||
format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval;
|
||||
format_settings.avro.schema_registry_url = settings.format_avro_schema_registry_url.toString();
|
||||
format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes;
|
||||
format_settings.csv.unquoted_null_literal_as_null = settings.input_format_csv_unquoted_null_literal_as_null;
|
||||
format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes;
|
||||
format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line;
|
||||
format_settings.csv.delimiter = settings.format_csv_delimiter;
|
||||
format_settings.csv.empty_as_default = settings.input_format_defaults_for_omitted_fields;
|
||||
format_settings.csv.input_format_enum_as_number = settings.input_format_csv_enum_as_number;
|
||||
format_settings.null_as_default = settings.input_format_null_as_default;
|
||||
format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions;
|
||||
format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions;
|
||||
format_settings.values.accurate_types_of_literals = settings.input_format_values_accurate_types_of_literals;
|
||||
format_settings.with_names_use_header = settings.input_format_with_names_use_header;
|
||||
format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields;
|
||||
format_settings.import_nested_json = settings.input_format_import_nested_json;
|
||||
format_settings.csv.unquoted_null_literal_as_null = settings.input_format_csv_unquoted_null_literal_as_null;
|
||||
format_settings.custom.escaping_rule = settings.format_custom_escaping_rule;
|
||||
format_settings.custom.field_delimiter = settings.format_custom_field_delimiter;
|
||||
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
|
||||
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
|
||||
format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter;
|
||||
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
|
||||
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
|
||||
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
|
||||
format_settings.date_time_input_format = settings.date_time_input_format;
|
||||
format_settings.date_time_output_format = settings.date_time_output_format;
|
||||
format_settings.enable_streaming = settings.output_format_enable_streaming;
|
||||
format_settings.import_nested_json = settings.input_format_import_nested_json;
|
||||
format_settings.input_allow_errors_num = settings.input_format_allow_errors_num;
|
||||
format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio;
|
||||
format_settings.template_settings.resultset_format = settings.format_template_resultset;
|
||||
format_settings.template_settings.row_format = settings.format_template_row;
|
||||
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
|
||||
format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default;
|
||||
format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number;
|
||||
format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes;
|
||||
format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
|
||||
format_settings.json.quote_denormals = settings.output_format_json_quote_denormals;
|
||||
format_settings.null_as_default = settings.input_format_null_as_default;
|
||||
format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size;
|
||||
format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8;
|
||||
format_settings.pretty.color = settings.output_format_pretty_color;
|
||||
format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width;
|
||||
format_settings.pretty.max_rows = settings.output_format_pretty_max_rows;
|
||||
format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width;
|
||||
format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers;
|
||||
format_settings.regexp.escaping_rule = settings.format_regexp_escaping_rule;
|
||||
format_settings.regexp.regexp = settings.format_regexp;
|
||||
format_settings.regexp.skip_unmatched = settings.format_regexp_skip_unmatched;
|
||||
format_settings.schema.format_schema = settings.format_schema;
|
||||
format_settings.schema.format_schema_path = context.getFormatSchemaPath();
|
||||
format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
|
||||
format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter;
|
||||
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
|
||||
format_settings.custom.escaping_rule = settings.format_custom_escaping_rule;
|
||||
format_settings.custom.field_delimiter = settings.format_custom_field_delimiter;
|
||||
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
|
||||
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
|
||||
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
|
||||
format_settings.regexp.regexp = settings.format_regexp;
|
||||
format_settings.regexp.escaping_rule = settings.format_regexp_escaping_rule;
|
||||
format_settings.regexp.skip_unmatched = settings.format_regexp_skip_unmatched;
|
||||
format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields;
|
||||
format_settings.template_settings.resultset_format = settings.format_template_resultset;
|
||||
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
|
||||
format_settings.template_settings.row_format = settings.format_template_row;
|
||||
format_settings.tsv.crlf_end_of_line = settings.output_format_tsv_crlf_end_of_line;
|
||||
format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default;
|
||||
format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number;
|
||||
format_settings.tsv.null_representation = settings.output_format_tsv_null_representation;
|
||||
format_settings.values.accurate_types_of_literals = settings.input_format_values_accurate_types_of_literals;
|
||||
format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions;
|
||||
format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions;
|
||||
format_settings.with_names_use_header = settings.input_format_with_names_use_header;
|
||||
format_settings.write_statistics = settings.output_format_write_statistics;
|
||||
|
||||
/// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context
|
||||
if (context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER))
|
||||
if (format_settings.schema.is_server)
|
||||
{
|
||||
const Poco::URI & avro_schema_registry_url = settings.format_avro_schema_registry_url;
|
||||
if (!avro_schema_registry_url.empty())
|
||||
context.getRemoteHostFilter().checkURL(avro_schema_registry_url);
|
||||
}
|
||||
format_settings.avro.schema_registry_url = settings.format_avro_schema_registry_url.toString();
|
||||
format_settings.avro.allow_missing_fields = settings.input_format_avro_allow_missing_fields;
|
||||
|
||||
return format_settings;
|
||||
}
|
||||
|
||||
static FormatSettings getOutputFormatSetting(const Settings & settings, const Context & context)
|
||||
{
|
||||
FormatSettings format_settings;
|
||||
format_settings.enable_streaming = settings.output_format_enable_streaming;
|
||||
format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
|
||||
format_settings.json.quote_denormals = settings.output_format_json_quote_denormals;
|
||||
format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes;
|
||||
format_settings.csv.delimiter = settings.format_csv_delimiter;
|
||||
format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes;
|
||||
format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes;
|
||||
format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line;
|
||||
format_settings.pretty.max_rows = settings.output_format_pretty_max_rows;
|
||||
format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width;
|
||||
format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width;
|
||||
format_settings.pretty.color = settings.output_format_pretty_color;
|
||||
format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ?
|
||||
FormatSettings::Pretty::Charset::ASCII :
|
||||
FormatSettings::Pretty::Charset::UTF8;
|
||||
format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers;
|
||||
format_settings.template_settings.resultset_format = settings.format_template_resultset;
|
||||
format_settings.template_settings.row_format = settings.format_template_row;
|
||||
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
|
||||
format_settings.tsv.crlf_end_of_line = settings.output_format_tsv_crlf_end_of_line;
|
||||
format_settings.tsv.null_representation = settings.output_format_tsv_null_representation;
|
||||
format_settings.write_statistics = settings.output_format_write_statistics;
|
||||
format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size;
|
||||
format_settings.schema.format_schema = settings.format_schema;
|
||||
format_settings.schema.format_schema_path = context.getFormatSchemaPath();
|
||||
format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
|
||||
format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter;
|
||||
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
|
||||
format_settings.custom.escaping_rule = settings.format_custom_escaping_rule;
|
||||
format_settings.custom.field_delimiter = settings.format_custom_field_delimiter;
|
||||
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
|
||||
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
|
||||
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
|
||||
format_settings.avro.output_codec = settings.output_format_avro_codec;
|
||||
format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval;
|
||||
format_settings.date_time_output_format = settings.date_time_output_format;
|
||||
template
|
||||
FormatSettings getFormatSettings<FormatFactorySettings>(const Context & context,
|
||||
const FormatFactorySettings & settings);
|
||||
|
||||
return format_settings;
|
||||
}
|
||||
template
|
||||
FormatSettings getFormatSettings<Settings>(const Context & context,
|
||||
const Settings & settings);
|
||||
|
||||
|
||||
BlockInputStreamPtr FormatFactory::getInput(
|
||||
@ -142,21 +135,22 @@ BlockInputStreamPtr FormatFactory::getInput(
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
UInt64 max_block_size,
|
||||
ReadCallback callback) const
|
||||
const std::optional<FormatSettings> & _format_settings) const
|
||||
{
|
||||
if (name == "Native")
|
||||
return std::make_shared<NativeBlockInputStream>(buf, sample, 0);
|
||||
|
||||
auto format_settings = _format_settings
|
||||
? *_format_settings : getFormatSettings(context);
|
||||
|
||||
if (!getCreators(name).input_processor_creator)
|
||||
{
|
||||
const auto & input_getter = getCreators(name).input_creator;
|
||||
if (!input_getter)
|
||||
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
FormatSettings format_settings = getInputFormatSetting(settings, context);
|
||||
|
||||
return input_getter(buf, sample, max_block_size, callback ? callback : ReadCallback(), format_settings);
|
||||
return input_getter(buf, sample, max_block_size, {}, format_settings);
|
||||
}
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
@ -182,17 +176,16 @@ BlockInputStreamPtr FormatFactory::getInput(
|
||||
if (!input_getter)
|
||||
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
|
||||
|
||||
FormatSettings format_settings = getInputFormatSetting(settings, context);
|
||||
|
||||
RowInputFormatParams row_input_format_params;
|
||||
row_input_format_params.max_block_size = max_block_size;
|
||||
row_input_format_params.allow_errors_num = format_settings.input_allow_errors_num;
|
||||
row_input_format_params.allow_errors_ratio = format_settings.input_allow_errors_ratio;
|
||||
row_input_format_params.callback = std::move(callback);
|
||||
row_input_format_params.max_execution_time = settings.max_execution_time;
|
||||
row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode;
|
||||
|
||||
auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, row_input_format_params, format_settings};
|
||||
auto input_creator_params =
|
||||
ParallelParsingBlockInputStream::InputCreatorParams{sample,
|
||||
row_input_format_params, format_settings};
|
||||
ParallelParsingBlockInputStream::Params params{buf, input_getter,
|
||||
input_creator_params, file_segmentation_engine,
|
||||
static_cast<int>(settings.max_threads),
|
||||
@ -200,32 +193,37 @@ BlockInputStreamPtr FormatFactory::getInput(
|
||||
return std::make_shared<ParallelParsingBlockInputStream>(params);
|
||||
}
|
||||
|
||||
auto format = getInputFormat(name, buf, sample, context, max_block_size, std::move(callback));
|
||||
auto format = getInputFormat(name, buf, sample, context, max_block_size,
|
||||
format_settings);
|
||||
return std::make_shared<InputStreamFromInputFormat>(std::move(format));
|
||||
}
|
||||
|
||||
|
||||
BlockOutputStreamPtr FormatFactory::getOutput(
|
||||
const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const bool ignore_no_row_delimiter) const
|
||||
BlockOutputStreamPtr FormatFactory::getOutput(const String & name,
|
||||
WriteBuffer & buf, const Block & sample, const Context & context,
|
||||
WriteCallback callback, const std::optional<FormatSettings> & _format_settings) const
|
||||
{
|
||||
auto format_settings = _format_settings
|
||||
? *_format_settings : getFormatSettings(context);
|
||||
|
||||
if (!getCreators(name).output_processor_creator)
|
||||
{
|
||||
const auto & output_getter = getCreators(name).output_creator;
|
||||
if (!output_getter)
|
||||
throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
FormatSettings format_settings = getOutputFormatSetting(settings, context);
|
||||
|
||||
/** Materialization is needed, because formats can use the functions `IDataType`,
|
||||
* which only work with full columns.
|
||||
*/
|
||||
return std::make_shared<MaterializingBlockOutputStream>(
|
||||
output_getter(buf, sample, std::move(callback), format_settings), sample);
|
||||
output_getter(buf, sample, std::move(callback), format_settings),
|
||||
sample);
|
||||
}
|
||||
|
||||
auto format = getOutputFormat(name, buf, sample, context, std::move(callback), ignore_no_row_delimiter);
|
||||
return std::make_shared<MaterializingBlockOutputStream>(std::make_shared<OutputStreamToOutputFormat>(format), sample);
|
||||
auto format = getOutputFormat(name, buf, sample, context, std::move(callback),
|
||||
format_settings);
|
||||
return std::make_shared<MaterializingBlockOutputStream>(
|
||||
std::make_shared<OutputStreamToOutputFormat>(format), sample);
|
||||
}
|
||||
|
||||
|
||||
@ -235,25 +233,27 @@ InputFormatPtr FormatFactory::getInputFormat(
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
UInt64 max_block_size,
|
||||
ReadCallback callback) const
|
||||
const std::optional<FormatSettings> & _format_settings) const
|
||||
{
|
||||
const auto & input_getter = getCreators(name).input_processor_creator;
|
||||
if (!input_getter)
|
||||
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
FormatSettings format_settings = getInputFormatSetting(settings, context);
|
||||
|
||||
auto format_settings = _format_settings
|
||||
? *_format_settings : getFormatSettings(context);
|
||||
|
||||
RowInputFormatParams params;
|
||||
params.max_block_size = max_block_size;
|
||||
params.allow_errors_num = format_settings.input_allow_errors_num;
|
||||
params.allow_errors_ratio = format_settings.input_allow_errors_ratio;
|
||||
params.callback = std::move(callback);
|
||||
params.max_execution_time = settings.max_execution_time;
|
||||
params.timeout_overflow_mode = settings.timeout_overflow_mode;
|
||||
|
||||
auto format = input_getter(buf, sample, params, format_settings);
|
||||
|
||||
|
||||
/// It's a kludge. Because I cannot remove context from values format.
|
||||
if (auto * values = typeid_cast<ValuesBlockInputFormat *>(format.get()))
|
||||
values->setContext(context);
|
||||
@ -263,19 +263,20 @@ InputFormatPtr FormatFactory::getInputFormat(
|
||||
|
||||
|
||||
OutputFormatPtr FormatFactory::getOutputFormat(
|
||||
const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const bool ignore_no_row_delimiter) const
|
||||
const String & name, WriteBuffer & buf, const Block & sample,
|
||||
const Context & context, WriteCallback callback,
|
||||
const std::optional<FormatSettings> & _format_settings) const
|
||||
{
|
||||
const auto & output_getter = getCreators(name).output_processor_creator;
|
||||
if (!output_getter)
|
||||
throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
FormatSettings format_settings = getOutputFormatSetting(settings, context);
|
||||
|
||||
RowOutputFormatParams params;
|
||||
params.ignore_no_row_delimiter = ignore_no_row_delimiter;
|
||||
params.callback = std::move(callback);
|
||||
|
||||
auto format_settings = _format_settings
|
||||
? *_format_settings : getFormatSettings(context);
|
||||
|
||||
/** TODO: Materialization is needed, because formats can use the functions `IDataType`,
|
||||
* which only work with full columns.
|
||||
*/
|
||||
|
@ -3,6 +3,7 @@
|
||||
#include <common/types.h>
|
||||
#include <Columns/IColumn.h>
|
||||
#include <DataStreams/IBlockStream_fwd.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
#include <IO/BufferWithOwnMemory.h>
|
||||
|
||||
#include <functional>
|
||||
@ -16,6 +17,8 @@ namespace DB
|
||||
class Block;
|
||||
class Context;
|
||||
struct FormatSettings;
|
||||
struct Settings;
|
||||
struct FormatFactorySettings;
|
||||
|
||||
class ReadBuffer;
|
||||
class WriteBuffer;
|
||||
@ -32,6 +35,11 @@ struct RowOutputFormatParams;
|
||||
using InputFormatPtr = std::shared_ptr<IInputFormat>;
|
||||
using OutputFormatPtr = std::shared_ptr<IOutputFormat>;
|
||||
|
||||
FormatSettings getFormatSettings(const Context & context);
|
||||
|
||||
template <typename T>
|
||||
FormatSettings getFormatSettings(const Context & context,
|
||||
const T & settings);
|
||||
|
||||
/** Allows to create an IBlockInputStream or IBlockOutputStream by the name of the format.
|
||||
* Note: format and compression are independent things.
|
||||
@ -104,10 +112,11 @@ public:
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
UInt64 max_block_size,
|
||||
ReadCallback callback = {}) const;
|
||||
const std::optional<FormatSettings> & format_settings = std::nullopt) const;
|
||||
|
||||
BlockOutputStreamPtr getOutput(const String & name, WriteBuffer & buf,
|
||||
const Block & sample, const Context & context, WriteCallback callback = {}, const bool ignore_no_row_delimiter = false) const;
|
||||
const Block & sample, const Context & context, WriteCallback callback = {},
|
||||
const std::optional<FormatSettings> & format_settings = std::nullopt) const;
|
||||
|
||||
InputFormatPtr getInputFormat(
|
||||
const String & name,
|
||||
@ -115,10 +124,12 @@ public:
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
UInt64 max_block_size,
|
||||
ReadCallback callback = {}) const;
|
||||
const std::optional<FormatSettings> & format_settings = std::nullopt) const;
|
||||
|
||||
OutputFormatPtr getOutputFormat(
|
||||
const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, const bool ignore_no_row_delimiter = false) const;
|
||||
const String & name, WriteBuffer & buf, const Block & sample,
|
||||
const Context & context, WriteCallback callback = {},
|
||||
const std::optional<FormatSettings> & format_settings = std::nullopt) const;
|
||||
|
||||
/// Register format by its name.
|
||||
void registerInputFormat(const String & name, InputCreator input_creator);
|
||||
|
@ -6,10 +6,16 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Various tweaks for input/output formats.
|
||||
* Text serialization/deserialization of data types also depend on some of these settings.
|
||||
* NOTE Parameters for unrelated formats and unrelated data types
|
||||
* are collected in this struct - it prevents modularity, but they are difficult to separate.
|
||||
/**
|
||||
* Various tweaks for input/output formats. Text serialization/deserialization
|
||||
* of data types also depend on some of these settings. It is different from
|
||||
* FormatFactorySettings in that it has all necessary user-provided settings
|
||||
* combined with information from context etc, that we can use directly during
|
||||
* serialization. In contrast, FormatFactorySettings' job is to reflect the
|
||||
* changes made to user-visible format settings, such as when tweaking the
|
||||
* the format for File engine.
|
||||
* NOTE Parameters for unrelated formats and unrelated data types are collected
|
||||
* in this struct - it prevents modularity, but they are difficult to separate.
|
||||
*/
|
||||
struct FormatSettings
|
||||
{
|
||||
@ -17,76 +23,6 @@ struct FormatSettings
|
||||
/// Option means that each chunk of data need to be formatted independently. Also each chunk will be flushed at the end of processing.
|
||||
bool enable_streaming = false;
|
||||
|
||||
struct JSON
|
||||
{
|
||||
bool quote_64bit_integers = true;
|
||||
bool quote_denormals = true;
|
||||
bool escape_forward_slashes = true;
|
||||
};
|
||||
|
||||
JSON json;
|
||||
|
||||
struct CSV
|
||||
{
|
||||
char delimiter = ',';
|
||||
bool allow_single_quotes = true;
|
||||
bool allow_double_quotes = true;
|
||||
bool unquoted_null_literal_as_null = false;
|
||||
bool empty_as_default = false;
|
||||
bool crlf_end_of_line = false;
|
||||
bool input_format_enum_as_number = false;
|
||||
};
|
||||
|
||||
CSV csv;
|
||||
|
||||
struct Pretty
|
||||
{
|
||||
UInt64 max_rows = 10000;
|
||||
UInt64 max_column_pad_width = 250;
|
||||
UInt64 max_value_width = 10000;
|
||||
bool color = true;
|
||||
|
||||
bool output_format_pretty_row_numbers = false;
|
||||
|
||||
enum class Charset
|
||||
{
|
||||
UTF8,
|
||||
ASCII,
|
||||
};
|
||||
|
||||
Charset charset = Charset::UTF8;
|
||||
};
|
||||
|
||||
Pretty pretty;
|
||||
|
||||
struct Values
|
||||
{
|
||||
bool interpret_expressions = true;
|
||||
bool deduce_templates_of_expressions = true;
|
||||
bool accurate_types_of_literals = true;
|
||||
};
|
||||
|
||||
Values values;
|
||||
|
||||
struct Template
|
||||
{
|
||||
String resultset_format;
|
||||
String row_format;
|
||||
String row_between_delimiter;
|
||||
};
|
||||
|
||||
Template template_settings;
|
||||
|
||||
struct TSV
|
||||
{
|
||||
bool empty_as_default = false;
|
||||
bool crlf_end_of_line = false;
|
||||
String null_representation = "\\N";
|
||||
bool input_format_enum_as_number = false;
|
||||
};
|
||||
|
||||
TSV tsv;
|
||||
|
||||
bool skip_unknown_fields = false;
|
||||
bool with_names_use_header = false;
|
||||
bool write_statistics = true;
|
||||
@ -113,24 +49,29 @@ struct FormatSettings
|
||||
UInt64 input_allow_errors_num = 0;
|
||||
Float32 input_allow_errors_ratio = 0;
|
||||
|
||||
struct Arrow
|
||||
struct
|
||||
{
|
||||
UInt64 row_group_size = 1000000;
|
||||
} arrow;
|
||||
|
||||
struct Parquet
|
||||
struct
|
||||
{
|
||||
UInt64 row_group_size = 1000000;
|
||||
} parquet;
|
||||
String schema_registry_url;
|
||||
String output_codec;
|
||||
UInt64 output_sync_interval = 16 * 1024;
|
||||
bool allow_missing_fields = false;
|
||||
} avro;
|
||||
|
||||
struct Schema
|
||||
struct CSV
|
||||
{
|
||||
std::string format_schema;
|
||||
std::string format_schema_path;
|
||||
bool is_server = false;
|
||||
};
|
||||
|
||||
Schema schema;
|
||||
char delimiter = ',';
|
||||
bool allow_single_quotes = true;
|
||||
bool allow_double_quotes = true;
|
||||
bool unquoted_null_literal_as_null = false;
|
||||
bool empty_as_default = false;
|
||||
bool crlf_end_of_line = false;
|
||||
bool input_format_enum_as_number = false;
|
||||
} csv;
|
||||
|
||||
struct Custom
|
||||
{
|
||||
@ -141,29 +82,87 @@ struct FormatSettings
|
||||
std::string row_between_delimiter;
|
||||
std::string field_delimiter;
|
||||
std::string escaping_rule;
|
||||
};
|
||||
} custom;
|
||||
|
||||
Custom custom;
|
||||
|
||||
struct Avro
|
||||
struct
|
||||
{
|
||||
String schema_registry_url;
|
||||
String output_codec;
|
||||
UInt64 output_sync_interval = 16 * 1024;
|
||||
bool allow_missing_fields = false;
|
||||
};
|
||||
bool quote_64bit_integers = true;
|
||||
bool quote_denormals = true;
|
||||
bool escape_forward_slashes = true;
|
||||
bool serialize_as_strings = false;
|
||||
} json;
|
||||
|
||||
Avro avro;
|
||||
struct
|
||||
{
|
||||
UInt64 row_group_size = 1000000;
|
||||
} parquet;
|
||||
|
||||
struct Regexp
|
||||
struct Pretty
|
||||
{
|
||||
UInt64 max_rows = 10000;
|
||||
UInt64 max_column_pad_width = 250;
|
||||
UInt64 max_value_width = 10000;
|
||||
bool color = true;
|
||||
|
||||
bool output_format_pretty_row_numbers = false;
|
||||
|
||||
enum class Charset
|
||||
{
|
||||
UTF8,
|
||||
ASCII,
|
||||
};
|
||||
|
||||
Charset charset = Charset::UTF8;
|
||||
} pretty;
|
||||
|
||||
struct
|
||||
{
|
||||
bool write_row_delimiters = true;
|
||||
/**
|
||||
* Some buffers (kafka / rabbit) split the rows internally using callback,
|
||||
* and always send one row per message, so we can push there formats
|
||||
* without framing / delimiters (like ProtobufSingle). In other cases,
|
||||
* we have to enforce exporting at most one row in the format output,
|
||||
* because Protobuf without delimiters is not generally useful.
|
||||
*/
|
||||
bool allow_many_rows_no_delimiters = false;
|
||||
} protobuf;
|
||||
|
||||
struct
|
||||
{
|
||||
std::string regexp;
|
||||
std::string escaping_rule;
|
||||
bool skip_unmatched = false;
|
||||
};
|
||||
} regexp;
|
||||
|
||||
Regexp regexp;
|
||||
struct
|
||||
{
|
||||
std::string format_schema;
|
||||
std::string format_schema_path;
|
||||
bool is_server = false;
|
||||
} schema;
|
||||
|
||||
struct
|
||||
{
|
||||
String resultset_format;
|
||||
String row_format;
|
||||
String row_between_delimiter;
|
||||
} template_settings;
|
||||
|
||||
struct
|
||||
{
|
||||
bool empty_as_default = false;
|
||||
bool crlf_end_of_line = false;
|
||||
String null_representation = "\\N";
|
||||
bool input_format_enum_as_number = false;
|
||||
} tsv;
|
||||
|
||||
struct
|
||||
{
|
||||
bool interpret_expressions = true;
|
||||
bool deduce_templates_of_expressions = true;
|
||||
bool accurate_types_of_literals = true;
|
||||
} values;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -38,8 +38,8 @@ try
|
||||
|
||||
FormatSettings format_settings;
|
||||
|
||||
RowInputFormatParams in_params{DEFAULT_INSERT_BLOCK_SIZE, 0, 0, []{}};
|
||||
RowOutputFormatParams out_params{[](const Columns & /* columns */, size_t /* row */){},false};
|
||||
RowInputFormatParams in_params{DEFAULT_INSERT_BLOCK_SIZE, 0, 0};
|
||||
RowOutputFormatParams out_params{[](const Columns & /* columns */, size_t /* row */){}};
|
||||
|
||||
InputFormatPtr input_format = std::make_shared<TabSeparatedRowInputFormat>(sample, in_buf, in_params, false, false, format_settings);
|
||||
BlockInputStreamPtr block_input = std::make_shared<InputStreamFromInputFormat>(std::move(input_format));
|
||||
|
@ -817,7 +817,11 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
|
||||
{
|
||||
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
|
||||
|
||||
/// YYYY-MM-DD hh:mm:ss
|
||||
static constexpr auto date_time_broken_down_length = 19;
|
||||
/// YYYY-MM-DD
|
||||
static constexpr auto date_broken_down_length = 10;
|
||||
/// unix timestamp max length
|
||||
static constexpr auto unix_timestamp_max_length = 10;
|
||||
|
||||
char s[date_time_broken_down_length];
|
||||
@ -831,12 +835,15 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
|
||||
++buf.position();
|
||||
}
|
||||
|
||||
/// 2015-01-01 01:02:03
|
||||
/// 2015-01-01 01:02:03 or 2015-01-01
|
||||
if (s_pos == s + 4 && !buf.eof() && (*buf.position() < '0' || *buf.position() > '9'))
|
||||
{
|
||||
const size_t remaining_size = date_time_broken_down_length - (s_pos - s);
|
||||
size_t size = buf.read(s_pos, remaining_size);
|
||||
if (remaining_size != size)
|
||||
const auto already_read_length = s_pos - s;
|
||||
const size_t remaining_date_time_size = date_time_broken_down_length - already_read_length;
|
||||
const size_t remaining_date_size = date_broken_down_length - already_read_length;
|
||||
|
||||
size_t size = buf.read(s_pos, remaining_date_time_size);
|
||||
if (size != remaining_date_time_size && size != remaining_date_size)
|
||||
{
|
||||
s_pos[size] = 0;
|
||||
|
||||
@ -850,9 +857,16 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
|
||||
UInt8 month = (s[5] - '0') * 10 + (s[6] - '0');
|
||||
UInt8 day = (s[8] - '0') * 10 + (s[9] - '0');
|
||||
|
||||
UInt8 hour = (s[11] - '0') * 10 + (s[12] - '0');
|
||||
UInt8 minute = (s[14] - '0') * 10 + (s[15] - '0');
|
||||
UInt8 second = (s[17] - '0') * 10 + (s[18] - '0');
|
||||
UInt8 hour = 0;
|
||||
UInt8 minute = 0;
|
||||
UInt8 second = 0;
|
||||
|
||||
if (size == remaining_date_time_size)
|
||||
{
|
||||
hour = (s[11] - '0') * 10 + (s[12] - '0');
|
||||
minute = (s[14] - '0') * 10 + (s[15] - '0');
|
||||
second = (s[17] - '0') * 10 + (s[18] - '0');
|
||||
}
|
||||
|
||||
if (unlikely(year == 0))
|
||||
datetime = 0;
|
||||
|
@ -700,7 +700,7 @@ UInt128 stringToUUID(const String & str);
|
||||
template <typename ReturnType = void>
|
||||
ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut);
|
||||
|
||||
/** In YYYY-MM-DD hh:mm:ss format, according to specified time zone.
|
||||
/** In YYYY-MM-DD hh:mm:ss or YYYY-MM-DD format, according to specified time zone.
|
||||
* As an exception, also supported parsing of unix timestamp in form of decimal number.
|
||||
*/
|
||||
template <typename ReturnType = void>
|
||||
@ -709,12 +709,17 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
|
||||
/** Read 10 characters, that could represent unix timestamp.
|
||||
* Only unix timestamp of 5-10 characters is supported.
|
||||
* Then look at 5th character. If it is a number - treat whole as unix timestamp.
|
||||
* If it is not a number - then parse datetime in YYYY-MM-DD hh:mm:ss format.
|
||||
* If it is not a number - then parse datetime in YYYY-MM-DD hh:mm:ss or YYYY-MM-DD format.
|
||||
*/
|
||||
|
||||
/// Optimistic path, when whole value is in buffer.
|
||||
const char * s = buf.position();
|
||||
if (s + 19 <= buf.buffer().end())
|
||||
|
||||
/// YYYY-MM-DD hh:mm:ss
|
||||
static constexpr auto DateTimeStringInputSize = 19;
|
||||
bool optimistic_path_for_date_time_input = s + DateTimeStringInputSize <= buf.buffer().end();
|
||||
|
||||
if (optimistic_path_for_date_time_input)
|
||||
{
|
||||
if (s[4] < '0' || s[4] > '9')
|
||||
{
|
||||
@ -731,7 +736,7 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
|
||||
else
|
||||
datetime = date_lut.makeDateTime(year, month, day, hour, minute, second);
|
||||
|
||||
buf.position() += 19;
|
||||
buf.position() += DateTimeStringInputSize;
|
||||
return ReturnType(true);
|
||||
}
|
||||
else
|
||||
|
@ -87,7 +87,7 @@ BlockIO InterpreterAlterQuery::execute()
|
||||
if (!partition_commands.empty())
|
||||
{
|
||||
table->checkAlterPartitionIsPossible(partition_commands, metadata_snapshot, context.getSettingsRef());
|
||||
auto partition_commands_pipe = table->alterPartition(query_ptr, metadata_snapshot, partition_commands, context);
|
||||
auto partition_commands_pipe = table->alterPartition(metadata_snapshot, partition_commands, context);
|
||||
if (!partition_commands_pipe.empty())
|
||||
res.pipeline.init(std::move(partition_commands_pipe));
|
||||
}
|
||||
|
@ -5,6 +5,7 @@
|
||||
#include <Interpreters/MutationsInterpreter.h>
|
||||
#include <Interpreters/TreeRewriter.h>
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
#include <Storages/MergeTree/StorageFromMergeTreeDataPart.h>
|
||||
#include <Processors/Transforms/FilterTransform.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/Transforms/CreatingSetsTransform.h>
|
||||
@ -32,6 +33,7 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int UNKNOWN_MUTATION_COMMAND;
|
||||
@ -92,6 +94,7 @@ std::optional<String> findFirstNonDeterministicFunctionName(const MutationComman
|
||||
if (finder_data.nondeterministic_function_name)
|
||||
return finder_data.nondeterministic_function_name;
|
||||
|
||||
/// Currently UPDATE and DELETE both always have predicates so we can use fallthrough
|
||||
[[fallthrough]];
|
||||
}
|
||||
|
||||
@ -110,7 +113,7 @@ std::optional<String> findFirstNonDeterministicFunctionName(const MutationComman
|
||||
return {};
|
||||
}
|
||||
|
||||
ASTPtr prepareQueryAffectedAST(const std::vector<MutationCommand> & commands)
|
||||
ASTPtr prepareQueryAffectedAST(const std::vector<MutationCommand> & commands, const StoragePtr & storage, const Context & context)
|
||||
{
|
||||
/// Execute `SELECT count() FROM storage WHERE predicate1 OR predicate2 OR ...` query.
|
||||
/// The result can differ from the number of affected rows (e.g. if there is an UPDATE command that
|
||||
@ -125,20 +128,23 @@ ASTPtr prepareQueryAffectedAST(const std::vector<MutationCommand> & commands)
|
||||
count_func->arguments = std::make_shared<ASTExpressionList>();
|
||||
select->select()->children.push_back(count_func);
|
||||
|
||||
if (commands.size() == 1)
|
||||
select->setExpression(ASTSelectQuery::Expression::WHERE, commands[0].predicate->clone());
|
||||
else
|
||||
ASTs conditions;
|
||||
for (const MutationCommand & command : commands)
|
||||
{
|
||||
auto coalesced_predicates = std::make_shared<ASTFunction>();
|
||||
coalesced_predicates->name = "or";
|
||||
coalesced_predicates->arguments = std::make_shared<ASTExpressionList>();
|
||||
coalesced_predicates->children.push_back(coalesced_predicates->arguments);
|
||||
|
||||
for (const MutationCommand & command : commands)
|
||||
coalesced_predicates->arguments->children.push_back(command.predicate->clone());
|
||||
if (ASTPtr condition = getPartitionAndPredicateExpressionForMutationCommand(command, storage, context))
|
||||
conditions.push_back(std::move(condition));
|
||||
}
|
||||
|
||||
if (conditions.size() > 1)
|
||||
{
|
||||
auto coalesced_predicates = makeASTFunction("or");
|
||||
coalesced_predicates->arguments->children = std::move(conditions);
|
||||
select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(coalesced_predicates));
|
||||
}
|
||||
else if (conditions.size() == 1)
|
||||
{
|
||||
select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(conditions.front()));
|
||||
}
|
||||
|
||||
return select;
|
||||
}
|
||||
@ -167,8 +173,9 @@ ColumnDependencies getAllColumnDependencies(const StorageMetadataPtr & metadata_
|
||||
|
||||
}
|
||||
|
||||
|
||||
bool isStorageTouchedByMutations(
|
||||
StoragePtr storage,
|
||||
const StoragePtr & storage,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const std::vector<MutationCommand> & commands,
|
||||
Context context_copy)
|
||||
@ -176,16 +183,33 @@ bool isStorageTouchedByMutations(
|
||||
if (commands.empty())
|
||||
return false;
|
||||
|
||||
bool all_commands_can_be_skipped = true;
|
||||
auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast<StorageFromMergeTreeDataPart>(storage);
|
||||
for (const MutationCommand & command : commands)
|
||||
{
|
||||
if (!command.predicate) /// The command touches all rows.
|
||||
return true;
|
||||
|
||||
if (command.partition && !storage_from_merge_tree_data_part)
|
||||
throw Exception("ALTER UPDATE/DELETE ... IN PARTITION is not supported for non-MergeTree tables", ErrorCodes::NOT_IMPLEMENTED);
|
||||
|
||||
if (command.partition && storage_from_merge_tree_data_part)
|
||||
{
|
||||
const String partition_id = storage_from_merge_tree_data_part->getPartitionIDFromQuery(command.partition, context_copy);
|
||||
if (partition_id == storage_from_merge_tree_data_part->getPartitionId())
|
||||
all_commands_can_be_skipped = false;
|
||||
}
|
||||
else
|
||||
all_commands_can_be_skipped = false;
|
||||
}
|
||||
|
||||
if (all_commands_can_be_skipped)
|
||||
return false;
|
||||
|
||||
context_copy.setSetting("max_streams_to_max_threads_ratio", 1);
|
||||
context_copy.setSetting("max_threads", 1);
|
||||
|
||||
ASTPtr select_query = prepareQueryAffectedAST(commands);
|
||||
ASTPtr select_query = prepareQueryAffectedAST(commands, storage, context_copy);
|
||||
|
||||
/// Interpreter must be alive, when we use result of execute() method.
|
||||
/// For some reason it may copy context and and give it into ExpressionBlockInputStream
|
||||
@ -202,9 +226,42 @@ bool isStorageTouchedByMutations(
|
||||
|
||||
auto count = (*block.getByName("count()").column)[0].get<UInt64>();
|
||||
return count != 0;
|
||||
|
||||
}
|
||||
|
||||
|
||||
ASTPtr getPartitionAndPredicateExpressionForMutationCommand(
|
||||
const MutationCommand & command,
|
||||
const StoragePtr & storage,
|
||||
const Context & context
|
||||
)
|
||||
{
|
||||
ASTPtr partition_predicate_as_ast_func;
|
||||
if (command.partition)
|
||||
{
|
||||
String partition_id;
|
||||
|
||||
auto storage_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(storage);
|
||||
auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast<StorageFromMergeTreeDataPart>(storage);
|
||||
if (storage_merge_tree)
|
||||
partition_id = storage_merge_tree->getPartitionIDFromQuery(command.partition, context);
|
||||
else if (storage_from_merge_tree_data_part)
|
||||
partition_id = storage_from_merge_tree_data_part->getPartitionIDFromQuery(command.partition, context);
|
||||
else
|
||||
throw Exception("ALTER UPDATE/DELETE ... IN PARTITION is not supported for non-MergeTree tables", ErrorCodes::NOT_IMPLEMENTED);
|
||||
|
||||
partition_predicate_as_ast_func = makeASTFunction("equals",
|
||||
std::make_shared<ASTIdentifier>("_partition_id"),
|
||||
std::make_shared<ASTLiteral>(partition_id)
|
||||
);
|
||||
}
|
||||
|
||||
if (command.predicate && command.partition)
|
||||
return makeASTFunction("and", command.predicate->clone(), std::move(partition_predicate_as_ast_func));
|
||||
else
|
||||
return command.predicate ? command.predicate->clone() : partition_predicate_as_ast_func;
|
||||
}
|
||||
|
||||
|
||||
MutationsInterpreter::MutationsInterpreter(
|
||||
StoragePtr storage_,
|
||||
const StorageMetadataPtr & metadata_snapshot_,
|
||||
@ -349,7 +406,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
|
||||
if (stages.empty() || !stages.back().column_to_updated.empty())
|
||||
stages.emplace_back(context);
|
||||
|
||||
auto negated_predicate = makeASTFunction("isZeroOrNull", command.predicate->clone());
|
||||
auto negated_predicate = makeASTFunction("isZeroOrNull", getPartitionAndPredicateExpressionForMutationCommand(command));
|
||||
stages.back().filters.push_back(negated_predicate);
|
||||
}
|
||||
else if (command.type == MutationCommand::UPDATE)
|
||||
@ -387,7 +444,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
|
||||
const auto & update_expr = kv.second;
|
||||
auto updated_column = makeASTFunction("CAST",
|
||||
makeASTFunction("if",
|
||||
command.predicate->clone(),
|
||||
getPartitionAndPredicateExpressionForMutationCommand(command),
|
||||
makeASTFunction("CAST",
|
||||
update_expr->clone(),
|
||||
type_literal),
|
||||
@ -592,7 +649,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector<Stage> &
|
||||
for (const String & column : stage.output_columns)
|
||||
all_asts->children.push_back(std::make_shared<ASTIdentifier>(column));
|
||||
|
||||
auto syntax_result = TreeRewriter(context).analyze(all_asts, all_columns);
|
||||
auto syntax_result = TreeRewriter(context).analyze(all_asts, all_columns, storage, metadata_snapshot);
|
||||
if (context.hasQueryContext())
|
||||
for (const auto & it : syntax_result->getScalars())
|
||||
context.getQueryContext().addScalar(it.first, it.second);
|
||||
@ -759,10 +816,10 @@ const Block & MutationsInterpreter::getUpdatedHeader() const
|
||||
size_t MutationsInterpreter::evaluateCommandsSize()
|
||||
{
|
||||
for (const MutationCommand & command : commands)
|
||||
if (unlikely(!command.predicate)) /// The command touches all rows.
|
||||
if (unlikely(!command.predicate && !command.partition)) /// The command touches all rows.
|
||||
return mutation_ast->size();
|
||||
|
||||
return std::max(prepareQueryAffectedAST(commands)->size(), mutation_ast->size());
|
||||
return std::max(prepareQueryAffectedAST(commands, storage, context)->size(), mutation_ast->size());
|
||||
}
|
||||
|
||||
std::optional<SortDescription> MutationsInterpreter::getStorageSortDescriptionIfPossible(const Block & header) const
|
||||
@ -783,6 +840,11 @@ std::optional<SortDescription> MutationsInterpreter::getStorageSortDescriptionIf
|
||||
return sort_description;
|
||||
}
|
||||
|
||||
ASTPtr MutationsInterpreter::getPartitionAndPredicateExpressionForMutationCommand(const MutationCommand & command) const
|
||||
{
|
||||
return DB::getPartitionAndPredicateExpressionForMutationCommand(command, storage, context);
|
||||
}
|
||||
|
||||
bool MutationsInterpreter::Stage::isAffectingAllColumns(const Names & storage_columns) const
|
||||
{
|
||||
/// is subset
|
||||
|
@ -20,7 +20,17 @@ using QueryPipelinePtr = std::unique_ptr<QueryPipeline>;
|
||||
|
||||
/// Return false if the data isn't going to be changed by mutations.
|
||||
bool isStorageTouchedByMutations(
|
||||
StoragePtr storage, const StorageMetadataPtr & metadata_snapshot, const std::vector<MutationCommand> & commands, Context context_copy);
|
||||
const StoragePtr & storage,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const std::vector<MutationCommand> & commands,
|
||||
Context context_copy
|
||||
);
|
||||
|
||||
ASTPtr getPartitionAndPredicateExpressionForMutationCommand(
|
||||
const MutationCommand & command,
|
||||
const StoragePtr & storage,
|
||||
const Context & context
|
||||
);
|
||||
|
||||
/// Create an input stream that will read data from storage and apply mutation commands (UPDATEs, DELETEs, MATERIALIZEs)
|
||||
/// to this data.
|
||||
@ -59,6 +69,8 @@ private:
|
||||
|
||||
std::optional<SortDescription> getStorageSortDescriptionIfPossible(const Block & header) const;
|
||||
|
||||
ASTPtr getPartitionAndPredicateExpressionForMutationCommand(const MutationCommand & command) const;
|
||||
|
||||
StoragePtr storage;
|
||||
StorageMetadataPtr metadata_snapshot;
|
||||
MutationCommands commands;
|
||||
|
@ -90,7 +90,7 @@ void ASTAlterCommand::formatImpl(
|
||||
column->formatImpl(settings, state, frame);
|
||||
if (partition)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
partition->formatImpl(settings, state, frame);
|
||||
}
|
||||
}
|
||||
@ -150,7 +150,7 @@ void ASTAlterCommand::formatImpl(
|
||||
index->formatImpl(settings, state, frame);
|
||||
if (partition)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
partition->formatImpl(settings, state, frame);
|
||||
}
|
||||
}
|
||||
@ -161,7 +161,7 @@ void ASTAlterCommand::formatImpl(
|
||||
index->formatImpl(settings, state, frame);
|
||||
if (partition)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
partition->formatImpl(settings, state, frame);
|
||||
}
|
||||
}
|
||||
@ -178,7 +178,8 @@ void ASTAlterCommand::formatImpl(
|
||||
}
|
||||
else if (type == ASTAlterCommand::DROP_PARTITION)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << (detach ? "DETACH" : "DROP") << " PARTITION "
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str
|
||||
<< (detach ? "DETACH" : "DROP") << (part ? " PART " : " PARTITION ")
|
||||
<< (settings.hilite ? hilite_none : "");
|
||||
partition->formatImpl(settings, state, frame);
|
||||
}
|
||||
@ -271,7 +272,15 @@ void ASTAlterCommand::formatImpl(
|
||||
}
|
||||
else if (type == ASTAlterCommand::DELETE)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DELETE WHERE " << (settings.hilite ? hilite_none : "");
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DELETE" << (settings.hilite ? hilite_none : "");
|
||||
|
||||
if (partition)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
partition->formatImpl(settings, state, frame);
|
||||
}
|
||||
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : "");
|
||||
predicate->formatImpl(settings, state, frame);
|
||||
}
|
||||
else if (type == ASTAlterCommand::UPDATE)
|
||||
@ -279,6 +288,12 @@ void ASTAlterCommand::formatImpl(
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "UPDATE " << (settings.hilite ? hilite_none : "");
|
||||
update_assignments->formatImpl(settings, state, frame);
|
||||
|
||||
if (partition)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
partition->formatImpl(settings, state, frame);
|
||||
}
|
||||
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : "");
|
||||
predicate->formatImpl(settings, state, frame);
|
||||
}
|
||||
@ -297,7 +312,7 @@ void ASTAlterCommand::formatImpl(
|
||||
<< (settings.hilite ? hilite_none : "");
|
||||
if (partition)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
partition->formatImpl(settings, state, frame);
|
||||
}
|
||||
}
|
||||
|
@ -103,7 +103,7 @@ public:
|
||||
*/
|
||||
ASTPtr constraint;
|
||||
|
||||
/** Used in DROP PARTITION and ATTACH PARTITION FROM queries.
|
||||
/** Used in DROP PARTITION, ATTACH PARTITION FROM, UPDATE, DELETE queries.
|
||||
* The value or ID of the partition is stored here.
|
||||
*/
|
||||
ASTPtr partition;
|
||||
|
@ -55,6 +55,12 @@ const char * ParserComparisonExpression::operators[] =
|
||||
nullptr
|
||||
};
|
||||
|
||||
const char * ParserComparisonExpression::overlapping_operators_to_skip[] =
|
||||
{
|
||||
"IN PARTITION",
|
||||
nullptr
|
||||
};
|
||||
|
||||
const char * ParserLogicalNotExpression::operators[] =
|
||||
{
|
||||
"NOT", "not",
|
||||
@ -137,6 +143,14 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node
|
||||
/// try to find any of the valid operators
|
||||
|
||||
const char ** it;
|
||||
Expected stub;
|
||||
for (it = overlapping_operators_to_skip; *it; ++it)
|
||||
if (ParserKeyword{*it}.checkWithoutMoving(pos, stub))
|
||||
break;
|
||||
|
||||
if (*it)
|
||||
break;
|
||||
|
||||
for (it = operators; *it; it += 2)
|
||||
if (parseOperator(pos, *it, expected))
|
||||
break;
|
||||
|
@ -82,6 +82,7 @@ class ParserLeftAssociativeBinaryOperatorList : public IParserBase
|
||||
{
|
||||
private:
|
||||
Operators_t operators;
|
||||
Operators_t overlapping_operators_to_skip = { (const char *[]){ nullptr } };
|
||||
ParserPtr first_elem_parser;
|
||||
ParserPtr remaining_elem_parser;
|
||||
|
||||
@ -93,6 +94,11 @@ public:
|
||||
{
|
||||
}
|
||||
|
||||
ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, Operators_t overlapping_operators_to_skip_, ParserPtr && first_elem_parser_)
|
||||
: operators(operators_), overlapping_operators_to_skip(overlapping_operators_to_skip_), first_elem_parser(std::move(first_elem_parser_))
|
||||
{
|
||||
}
|
||||
|
||||
ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, ParserPtr && first_elem_parser_,
|
||||
ParserPtr && remaining_elem_parser_)
|
||||
: operators(operators_), first_elem_parser(std::move(first_elem_parser_)),
|
||||
@ -284,7 +290,8 @@ class ParserComparisonExpression : public IParserBase
|
||||
{
|
||||
private:
|
||||
static const char * operators[];
|
||||
ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique<ParserBetweenExpression>()};
|
||||
static const char * overlapping_operators_to_skip[];
|
||||
ParserLeftAssociativeBinaryOperatorList operator_parser {operators, overlapping_operators_to_skip, std::make_unique<ParserBetweenExpression>()};
|
||||
|
||||
protected:
|
||||
const char * getName() const override{ return "comparison expression"; }
|
||||
|
@ -52,13 +52,15 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
ParserKeyword s_modify("MODIFY");
|
||||
|
||||
ParserKeyword s_attach_partition("ATTACH PARTITION");
|
||||
ParserKeyword s_attach_part("ATTACH PART");
|
||||
ParserKeyword s_detach_partition("DETACH PARTITION");
|
||||
ParserKeyword s_detach_part("DETACH PART");
|
||||
ParserKeyword s_drop_partition("DROP PARTITION");
|
||||
ParserKeyword s_drop_part("DROP PART");
|
||||
ParserKeyword s_move_partition("MOVE PARTITION");
|
||||
ParserKeyword s_move_part("MOVE PART");
|
||||
ParserKeyword s_drop_detached_partition("DROP DETACHED PARTITION");
|
||||
ParserKeyword s_drop_detached_part("DROP DETACHED PART");
|
||||
ParserKeyword s_attach_part("ATTACH PART");
|
||||
ParserKeyword s_move_part("MOVE PART");
|
||||
ParserKeyword s_fetch_partition("FETCH PARTITION");
|
||||
ParserKeyword s_replace_partition("REPLACE PARTITION");
|
||||
ParserKeyword s_freeze("FREEZE");
|
||||
@ -77,7 +79,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
ParserKeyword s_to_volume("TO VOLUME");
|
||||
ParserKeyword s_to_table("TO TABLE");
|
||||
|
||||
ParserKeyword s_delete_where("DELETE WHERE");
|
||||
ParserKeyword s_delete("DELETE");
|
||||
ParserKeyword s_update("UPDATE");
|
||||
ParserKeyword s_where("WHERE");
|
||||
ParserKeyword s_to("TO");
|
||||
@ -161,6 +163,14 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
|
||||
command->type = ASTAlterCommand::DROP_PARTITION;
|
||||
}
|
||||
else if (s_drop_part.ignore(pos, expected))
|
||||
{
|
||||
if (!parser_string_literal.parse(pos, command->partition, expected))
|
||||
return false;
|
||||
|
||||
command->type = ASTAlterCommand::DROP_PARTITION;
|
||||
command->part = true;
|
||||
}
|
||||
else if (s_drop_detached_partition.ignore(pos, expected))
|
||||
{
|
||||
if (!parser_partition.parse(pos, command->partition, expected))
|
||||
@ -354,6 +364,15 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
command->type = ASTAlterCommand::DROP_PARTITION;
|
||||
command->detach = true;
|
||||
}
|
||||
else if (s_detach_part.ignore(pos, expected))
|
||||
{
|
||||
if (!parser_string_literal.parse(pos, command->partition, expected))
|
||||
return false;
|
||||
|
||||
command->type = ASTAlterCommand::DROP_PARTITION;
|
||||
command->part = true;
|
||||
command->detach = true;
|
||||
}
|
||||
else if (s_attach_partition.ignore(pos, expected))
|
||||
{
|
||||
if (!parser_partition.parse(pos, command->partition, expected))
|
||||
@ -487,8 +506,17 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
|
||||
command->type = ASTAlterCommand::MODIFY_SAMPLE_BY;
|
||||
}
|
||||
else if (s_delete_where.ignore(pos, expected))
|
||||
else if (s_delete.ignore(pos, expected))
|
||||
{
|
||||
if (s_in_partition.ignore(pos, expected))
|
||||
{
|
||||
if (!parser_partition.parse(pos, command->partition, expected))
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!s_where.ignore(pos, expected))
|
||||
return false;
|
||||
|
||||
if (!parser_exp_elem.parse(pos, command->predicate, expected))
|
||||
return false;
|
||||
|
||||
@ -499,6 +527,12 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
if (!parser_assignment_list.parse(pos, command->update_assignments, expected))
|
||||
return false;
|
||||
|
||||
if (s_in_partition.ignore(pos, expected))
|
||||
{
|
||||
if (!parser_partition.parse(pos, command->partition, expected))
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!s_where.ignore(pos, expected))
|
||||
return false;
|
||||
|
||||
|
@ -10,7 +10,7 @@ namespace DB
|
||||
* ALTER TABLE [db.]name [ON CLUSTER cluster]
|
||||
* [ADD COLUMN [IF NOT EXISTS] col_name type [AFTER col_after],]
|
||||
* [DROP COLUMN [IF EXISTS] col_to_drop, ...]
|
||||
* [CLEAR COLUMN [IF EXISTS] col_to_clear [IN PARTITION partition],]
|
||||
* [CLEAR COLUMN [IF EXISTS] col_to_clear[ IN PARTITION partition],]
|
||||
* [MODIFY COLUMN [IF EXISTS] col_to_modify type, ...]
|
||||
* [RENAME COLUMN [IF EXISTS] col_name TO col_name]
|
||||
* [MODIFY PRIMARY KEY (a, b, c...)]
|
||||
@ -19,8 +19,12 @@ namespace DB
|
||||
* [DROP|DETACH|ATTACH PARTITION|PART partition, ...]
|
||||
* [FETCH PARTITION partition FROM ...]
|
||||
* [FREEZE [PARTITION] [WITH NAME name]]
|
||||
* [DELETE WHERE ...]
|
||||
* [UPDATE col_name = expr, ... WHERE ...]
|
||||
* [DELETE[ IN PARTITION partition] WHERE ...]
|
||||
* [UPDATE col_name = expr, ...[ IN PARTITION partition] WHERE ...]
|
||||
* [ADD INDEX [IF NOT EXISTS] index_name [AFTER index_name]]
|
||||
* [DROP INDEX [IF EXISTS] index_name]
|
||||
* [CLEAR INDEX [IF EXISTS] index_name IN PARTITION partition]
|
||||
* [MATERIALIZE INDEX [IF EXISTS] index_name [IN PARTITION partition]]
|
||||
* ALTER LIVE VIEW [db.name]
|
||||
* [REFRESH]
|
||||
*/
|
||||
|
@ -65,8 +65,6 @@ Chunk IRowInputFormat::generate()
|
||||
info.read_columns.clear();
|
||||
if (!readRow(columns, info))
|
||||
break;
|
||||
if (params.callback)
|
||||
params.callback();
|
||||
|
||||
for (size_t column_idx = 0; column_idx < info.read_columns.size(); ++column_idx)
|
||||
{
|
||||
|
@ -27,9 +27,6 @@ struct RowInputFormatParams
|
||||
UInt64 allow_errors_num;
|
||||
Float64 allow_errors_ratio;
|
||||
|
||||
using ReadCallback = std::function<void()>;
|
||||
ReadCallback callback;
|
||||
|
||||
Poco::Timespan max_execution_time = 0;
|
||||
OverflowMode timeout_overflow_mode = OverflowMode::THROW;
|
||||
};
|
||||
|
@ -15,14 +15,6 @@ struct RowOutputFormatParams
|
||||
|
||||
// Callback used to indicate that another row is written.
|
||||
WriteCallback callback;
|
||||
|
||||
/**
|
||||
* some buffers (kafka / rabbit) split the rows internally using callback
|
||||
* so we can push there formats without framing / delimiters
|
||||
* (like ProtobufSingle). In other cases you can't write more than single row
|
||||
* in unframed format.
|
||||
*/
|
||||
bool ignore_no_row_delimiter = false;
|
||||
};
|
||||
|
||||
class WriteBuffer;
|
||||
|
@ -23,18 +23,22 @@ ProtobufRowOutputFormat::ProtobufRowOutputFormat(
|
||||
const Block & header,
|
||||
const RowOutputFormatParams & params_,
|
||||
const FormatSchemaInfo & format_schema,
|
||||
const bool use_length_delimiters_)
|
||||
const FormatSettings & settings)
|
||||
: IRowOutputFormat(header, out_, params_)
|
||||
, data_types(header.getDataTypes())
|
||||
, writer(out, ProtobufSchemas::instance().getMessageTypeForFormatSchema(format_schema), header.getNames(), use_length_delimiters_)
|
||||
, throw_on_multiple_rows_undelimited(!use_length_delimiters_ && !params_.ignore_no_row_delimiter)
|
||||
, writer(out,
|
||||
ProtobufSchemas::instance().getMessageTypeForFormatSchema(format_schema),
|
||||
header.getNames(), settings.protobuf.write_row_delimiters)
|
||||
, allow_only_one_row(
|
||||
!settings.protobuf.write_row_delimiters
|
||||
&& !settings.protobuf.allow_many_rows_no_delimiters)
|
||||
{
|
||||
value_indices.resize(header.columns());
|
||||
}
|
||||
|
||||
void ProtobufRowOutputFormat::write(const Columns & columns, size_t row_num)
|
||||
{
|
||||
if (throw_on_multiple_rows_undelimited && !first_row)
|
||||
if (allow_only_one_row && !first_row)
|
||||
{
|
||||
throw Exception("The ProtobufSingle format can't be used to write multiple rows because this format doesn't have any row delimiter.", ErrorCodes::NO_ROW_DELIMITER);
|
||||
}
|
||||
@ -51,19 +55,23 @@ void ProtobufRowOutputFormat::write(const Columns & columns, size_t row_num)
|
||||
|
||||
void registerOutputFormatProcessorProtobuf(FormatFactory & factory)
|
||||
{
|
||||
for (bool use_length_delimiters : {false, true})
|
||||
for (bool write_row_delimiters : {false, true})
|
||||
{
|
||||
factory.registerOutputFormatProcessor(
|
||||
use_length_delimiters ? "Protobuf" : "ProtobufSingle",
|
||||
[use_length_delimiters](WriteBuffer & buf,
|
||||
write_row_delimiters ? "Protobuf" : "ProtobufSingle",
|
||||
[write_row_delimiters](WriteBuffer & buf,
|
||||
const Block & header,
|
||||
const RowOutputFormatParams & params,
|
||||
const FormatSettings & settings)
|
||||
const FormatSettings & _settings)
|
||||
{
|
||||
return std::make_shared<ProtobufRowOutputFormat>(buf, header, params,
|
||||
FormatSchemaInfo(settings.schema.format_schema, "Protobuf", true,
|
||||
settings.schema.is_server, settings.schema.format_schema_path),
|
||||
use_length_delimiters);
|
||||
FormatSettings settings = _settings;
|
||||
settings.protobuf.write_row_delimiters = write_row_delimiters;
|
||||
return std::make_shared<ProtobufRowOutputFormat>(
|
||||
buf, header, params,
|
||||
FormatSchemaInfo(settings.schema.format_schema, "Protobuf",
|
||||
true, settings.schema.is_server,
|
||||
settings.schema.format_schema_path),
|
||||
settings);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@ -41,7 +41,7 @@ public:
|
||||
const Block & header,
|
||||
const RowOutputFormatParams & params_,
|
||||
const FormatSchemaInfo & format_schema,
|
||||
const bool use_length_delimiters_);
|
||||
const FormatSettings & settings);
|
||||
|
||||
String getName() const override { return "ProtobufRowOutputFormat"; }
|
||||
|
||||
@ -53,7 +53,7 @@ private:
|
||||
DataTypes data_types;
|
||||
ProtobufWriter writer;
|
||||
std::vector<size_t> value_indices;
|
||||
const bool throw_on_multiple_rows_undelimited;
|
||||
const bool allow_only_one_row;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -54,8 +54,6 @@ Chunk ValuesBlockInputFormat::generate()
|
||||
if (buf.eof() || *buf.position() == ';')
|
||||
break;
|
||||
readRow(columns, rows_in_block);
|
||||
if (params.callback)
|
||||
params.callback();
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
|
@ -115,7 +115,6 @@ void IStorage::read(
|
||||
}
|
||||
|
||||
Pipe IStorage::alterPartition(
|
||||
const ASTPtr & /* query */,
|
||||
const StorageMetadataPtr & /* metadata_snapshot */,
|
||||
const PartitionCommands & /* commands */,
|
||||
const Context & /* context */)
|
||||
|
@ -367,7 +367,6 @@ public:
|
||||
* Should handle locks for each command on its own.
|
||||
*/
|
||||
virtual Pipe alterPartition(
|
||||
const ASTPtr & /* query */,
|
||||
const StorageMetadataPtr & /* metadata_snapshot */,
|
||||
const PartitionCommands & /* commands */,
|
||||
const Context & /* context */);
|
||||
|
@ -32,13 +32,16 @@ void KafkaBlockOutputStream::writePrefix()
|
||||
if (!buffer)
|
||||
throw Exception("Failed to create Kafka producer!", ErrorCodes::CANNOT_CREATE_IO_BUFFER);
|
||||
|
||||
child = FormatFactory::instance().getOutput(
|
||||
storage.getFormatName(), *buffer, getHeader(), *context, [this](const Columns & columns, size_t row)
|
||||
{
|
||||
buffer->countRow(columns, row);
|
||||
},
|
||||
/* ignore_no_row_delimiter = */ true
|
||||
);
|
||||
auto format_settings = getFormatSettings(*context);
|
||||
format_settings.protobuf.allow_many_rows_no_delimiters = true;
|
||||
|
||||
child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer,
|
||||
getHeader(), *context,
|
||||
[this](const Columns & columns, size_t row)
|
||||
{
|
||||
buffer->countRow(columns, row);
|
||||
},
|
||||
format_settings);
|
||||
}
|
||||
|
||||
void KafkaBlockOutputStream::write(const Block & block)
|
||||
|
@ -9,17 +9,17 @@ struct BoolMask
|
||||
BoolMask() {}
|
||||
BoolMask(bool can_be_true_, bool can_be_false_) : can_be_true(can_be_true_), can_be_false(can_be_false_) {}
|
||||
|
||||
BoolMask operator &(const BoolMask & m)
|
||||
BoolMask operator &(const BoolMask & m) const
|
||||
{
|
||||
return BoolMask(can_be_true && m.can_be_true, can_be_false || m.can_be_false);
|
||||
return {can_be_true && m.can_be_true, can_be_false || m.can_be_false};
|
||||
}
|
||||
BoolMask operator |(const BoolMask & m)
|
||||
BoolMask operator |(const BoolMask & m) const
|
||||
{
|
||||
return BoolMask(can_be_true || m.can_be_true, can_be_false && m.can_be_false);
|
||||
return {can_be_true || m.can_be_true, can_be_false && m.can_be_false};
|
||||
}
|
||||
BoolMask operator !()
|
||||
BoolMask operator !() const
|
||||
{
|
||||
return BoolMask(can_be_false, can_be_true);
|
||||
return {can_be_false, can_be_true};
|
||||
}
|
||||
|
||||
/// If mask is (true, true), then it can no longer change under operation |.
|
||||
|
@ -1,6 +1,7 @@
|
||||
#include <Storages/MergeTree/EphemeralLockInZooKeeper.h>
|
||||
#include <Common/ZooKeeper/KeeperException.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <common/types.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -71,13 +72,13 @@ EphemeralLockInZooKeeper::~EphemeralLockInZooKeeper()
|
||||
EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions(
|
||||
const String & block_numbers_path, const String & path_prefix, const String & temp_path,
|
||||
zkutil::ZooKeeper & zookeeper_)
|
||||
: zookeeper(zookeeper_)
|
||||
: zookeeper(&zookeeper_)
|
||||
{
|
||||
std::vector<String> holders;
|
||||
while (true)
|
||||
{
|
||||
Coordination::Stat partitions_stat;
|
||||
Strings partitions = zookeeper.getChildren(block_numbers_path, &partitions_stat);
|
||||
Strings partitions = zookeeper->getChildren(block_numbers_path, &partitions_stat);
|
||||
|
||||
if (holders.size() < partitions.size())
|
||||
{
|
||||
@ -85,7 +86,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions(
|
||||
for (size_t i = 0; i < partitions.size() - holders.size(); ++i)
|
||||
{
|
||||
String path = temp_path + "/abandonable_lock-";
|
||||
holder_futures.push_back(zookeeper.asyncCreate(path, {}, zkutil::CreateMode::EphemeralSequential));
|
||||
holder_futures.push_back(zookeeper->asyncCreate(path, {}, zkutil::CreateMode::EphemeralSequential));
|
||||
}
|
||||
for (auto & future : holder_futures)
|
||||
{
|
||||
@ -104,7 +105,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions(
|
||||
lock_ops.push_back(zkutil::makeCheckRequest(block_numbers_path, partitions_stat.version));
|
||||
|
||||
Coordination::Responses lock_responses;
|
||||
Coordination::Error rc = zookeeper.tryMulti(lock_ops, lock_responses);
|
||||
Coordination::Error rc = zookeeper->tryMulti(lock_ops, lock_responses);
|
||||
if (rc == Coordination::Error::ZBADVERSION)
|
||||
{
|
||||
LOG_TRACE(&Poco::Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry.");
|
||||
@ -131,13 +132,16 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions(
|
||||
|
||||
void EphemeralLocksInAllPartitions::unlock()
|
||||
{
|
||||
if (!zookeeper)
|
||||
return;
|
||||
|
||||
std::vector<zkutil::ZooKeeper::FutureMulti> futures;
|
||||
for (const auto & lock : locks)
|
||||
{
|
||||
Coordination::Requests unlock_ops;
|
||||
unlock_ops.emplace_back(zkutil::makeRemoveRequest(lock.path, -1));
|
||||
unlock_ops.emplace_back(zkutil::makeRemoveRequest(lock.holder_path, -1));
|
||||
futures.push_back(zookeeper.asyncMulti(unlock_ops));
|
||||
futures.push_back(zookeeper->asyncMulti(unlock_ops));
|
||||
}
|
||||
|
||||
for (auto & future : futures)
|
||||
|
@ -1,9 +1,14 @@
|
||||
#pragma once
|
||||
|
||||
#include "ReplicatedMergeTreeMutationEntry.h"
|
||||
|
||||
#include <Common/ZooKeeper/ZooKeeper.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
#include <map>
|
||||
#include <optional>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -87,13 +92,30 @@ private:
|
||||
|
||||
|
||||
/// Acquires block number locks in all partitions.
|
||||
class EphemeralLocksInAllPartitions : private boost::noncopyable
|
||||
class EphemeralLocksInAllPartitions : public boost::noncopyable
|
||||
{
|
||||
public:
|
||||
EphemeralLocksInAllPartitions(
|
||||
const String & block_numbers_path, const String & path_prefix, const String & temp_path,
|
||||
zkutil::ZooKeeper & zookeeper_);
|
||||
|
||||
EphemeralLocksInAllPartitions() = default;
|
||||
|
||||
EphemeralLocksInAllPartitions(EphemeralLocksInAllPartitions && rhs) noexcept
|
||||
: zookeeper(rhs.zookeeper)
|
||||
, locks(std::move(rhs.locks))
|
||||
{
|
||||
rhs.zookeeper = nullptr;
|
||||
}
|
||||
|
||||
EphemeralLocksInAllPartitions & operator=(EphemeralLocksInAllPartitions && rhs) noexcept
|
||||
{
|
||||
zookeeper = rhs.zookeeper;
|
||||
rhs.zookeeper = nullptr;
|
||||
locks = std::move(rhs.locks);
|
||||
return *this;
|
||||
}
|
||||
|
||||
struct LockInfo
|
||||
{
|
||||
String path;
|
||||
@ -110,8 +132,51 @@ public:
|
||||
~EphemeralLocksInAllPartitions();
|
||||
|
||||
private:
|
||||
zkutil::ZooKeeper & zookeeper;
|
||||
zkutil::ZooKeeper * zookeeper = nullptr;
|
||||
std::vector<LockInfo> locks;
|
||||
};
|
||||
|
||||
|
||||
/// This class allows scoped manipulations with block numbers locked in certain partitions
|
||||
/// See StorageReplicatedMergeTree::allocateBlockNumbersInAffectedPartitions and alter()/mutate() methods
|
||||
class PartitionBlockNumbersHolder
|
||||
{
|
||||
public:
|
||||
PartitionBlockNumbersHolder(const PartitionBlockNumbersHolder &) = delete;
|
||||
PartitionBlockNumbersHolder & operator=(const PartitionBlockNumbersHolder &) = delete;
|
||||
|
||||
using BlockNumbersType = ReplicatedMergeTreeMutationEntry::BlockNumbersType;
|
||||
|
||||
PartitionBlockNumbersHolder() = default;
|
||||
PartitionBlockNumbersHolder(
|
||||
BlockNumbersType block_numbers_, std::optional<EphemeralLocksInAllPartitions> locked_block_numbers_holder)
|
||||
: block_numbers(std::move(block_numbers_))
|
||||
, multiple_partitions_holder(std::move(locked_block_numbers_holder))
|
||||
{
|
||||
}
|
||||
PartitionBlockNumbersHolder(
|
||||
BlockNumbersType block_numbers_, std::optional<EphemeralLockInZooKeeper> locked_block_numbers_holder)
|
||||
: block_numbers(std::move(block_numbers_))
|
||||
, single_partition_holder(std::move(locked_block_numbers_holder))
|
||||
{
|
||||
}
|
||||
|
||||
PartitionBlockNumbersHolder & operator=(PartitionBlockNumbersHolder &&) = default;
|
||||
|
||||
const BlockNumbersType & getBlockNumbers() const { return block_numbers; }
|
||||
|
||||
void reset()
|
||||
{
|
||||
multiple_partitions_holder.reset();
|
||||
single_partition_holder.reset();
|
||||
block_numbers.clear();
|
||||
}
|
||||
|
||||
private:
|
||||
BlockNumbersType block_numbers;
|
||||
|
||||
std::optional<EphemeralLocksInAllPartitions> multiple_partitions_holder;
|
||||
std::optional<EphemeralLockInZooKeeper> single_partition_holder;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -2643,6 +2643,17 @@ void MergeTreeData::checkPartitionCanBeDropped(const ASTPtr & partition)
|
||||
global_context.checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, partition_size);
|
||||
}
|
||||
|
||||
void MergeTreeData::checkPartCanBeDropped(const ASTPtr & part_ast)
|
||||
{
|
||||
String part_name = part_ast->as<ASTLiteral &>().value.safeGet<String>();
|
||||
auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Committed});
|
||||
if (!part)
|
||||
throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in commited state", part_name);
|
||||
|
||||
auto table_id = getStorageID();
|
||||
global_context.checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, part->getBytesOnDisk());
|
||||
}
|
||||
|
||||
void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & name, bool moving_part, const Context & context)
|
||||
{
|
||||
String partition_id;
|
||||
|
@ -561,6 +561,8 @@ public:
|
||||
|
||||
void checkPartitionCanBeDropped(const ASTPtr & partition) override;
|
||||
|
||||
void checkPartCanBeDropped(const ASTPtr & part);
|
||||
|
||||
size_t getColumnCompressedSize(const std::string & name) const
|
||||
{
|
||||
auto lock = lockParts();
|
||||
|
@ -1,16 +1,17 @@
|
||||
#include <Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h>
|
||||
#include <Interpreters/misc.h>
|
||||
#include <Interpreters/BloomFilterHash.h>
|
||||
#include <Common/HashTable/ClearableHashMap.h>
|
||||
#include <Storages/MergeTree/RPNBuilder.h>
|
||||
#include <Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.h>
|
||||
#include <Common/FieldVisitorsAccurateComparison.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <ext/bit_cast.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Storages/MergeTree/RPNBuilder.h>
|
||||
#include <Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.h>
|
||||
#include <Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h>
|
||||
#include <Parsers/ASTSubquery.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Interpreters/misc.h>
|
||||
#include <Interpreters/BloomFilterHash.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include <Interpreters/convertFieldToType.h>
|
||||
|
||||
@ -105,11 +106,11 @@ bool MergeTreeIndexConditionBloomFilter::alwaysUnknownOrTrue() const
|
||||
rpn_stack.push_back(true);
|
||||
}
|
||||
else if (element.function == RPNElement::FUNCTION_EQUALS
|
||||
|| element.function == RPNElement::FUNCTION_NOT_EQUALS
|
||||
|| element.function == RPNElement::FUNCTION_HAS
|
||||
|| element.function == RPNElement::FUNCTION_IN
|
||||
|| element.function == RPNElement::FUNCTION_NOT_IN
|
||||
|| element.function == RPNElement::ALWAYS_FALSE)
|
||||
|| element.function == RPNElement::FUNCTION_NOT_EQUALS
|
||||
|| element.function == RPNElement::FUNCTION_HAS
|
||||
|| element.function == RPNElement::FUNCTION_IN
|
||||
|| element.function == RPNElement::FUNCTION_NOT_IN
|
||||
|| element.function == RPNElement::ALWAYS_FALSE)
|
||||
{
|
||||
rpn_stack.push_back(false);
|
||||
}
|
||||
@ -222,9 +223,21 @@ bool MergeTreeIndexConditionBloomFilter::traverseAtomAST(const ASTPtr & node, Bl
|
||||
}
|
||||
}
|
||||
|
||||
return traverseFunction(node, block_with_constants, out, nullptr);
|
||||
}
|
||||
|
||||
bool MergeTreeIndexConditionBloomFilter::traverseFunction(const ASTPtr & node, Block & block_with_constants, RPNElement & out, const ASTPtr & parent)
|
||||
{
|
||||
bool maybe_useful = false;
|
||||
|
||||
if (const auto * function = node->as<ASTFunction>())
|
||||
{
|
||||
const ASTs & arguments = function->arguments->children;
|
||||
for (const auto & arg : arguments)
|
||||
{
|
||||
if (traverseFunction(arg, block_with_constants, out, node))
|
||||
maybe_useful = true;
|
||||
}
|
||||
|
||||
if (arguments.size() != 2)
|
||||
return false;
|
||||
@ -232,20 +245,29 @@ bool MergeTreeIndexConditionBloomFilter::traverseAtomAST(const ASTPtr & node, Bl
|
||||
if (functionIsInOrGlobalInOperator(function->name))
|
||||
{
|
||||
if (const auto & prepared_set = getPreparedSet(arguments[1]))
|
||||
return traverseASTIn(function->name, arguments[0], prepared_set, out);
|
||||
{
|
||||
if (traverseASTIn(function->name, arguments[0], prepared_set, out))
|
||||
maybe_useful = true;
|
||||
}
|
||||
}
|
||||
else if (function->name == "equals" || function->name == "notEquals" || function->name == "has")
|
||||
else if (function->name == "equals" || function->name == "notEquals" || function->name == "has" || function->name == "indexOf")
|
||||
{
|
||||
Field const_value;
|
||||
DataTypePtr const_type;
|
||||
if (KeyCondition::getConstant(arguments[1], block_with_constants, const_value, const_type))
|
||||
return traverseASTEquals(function->name, arguments[0], const_type, const_value, out);
|
||||
{
|
||||
if (traverseASTEquals(function->name, arguments[0], const_type, const_value, out, parent))
|
||||
maybe_useful = true;
|
||||
}
|
||||
else if (KeyCondition::getConstant(arguments[0], block_with_constants, const_value, const_type))
|
||||
return traverseASTEquals(function->name, arguments[1], const_type, const_value, out);
|
||||
{
|
||||
if (traverseASTEquals(function->name, arguments[1], const_type, const_value, out, parent))
|
||||
maybe_useful = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
return maybe_useful;
|
||||
}
|
||||
|
||||
bool MergeTreeIndexConditionBloomFilter::traverseASTIn(
|
||||
@ -302,8 +324,66 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTIn(
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
static bool indexOfCanUseBloomFilter(const ASTPtr & parent)
|
||||
{
|
||||
if (!parent)
|
||||
return true;
|
||||
|
||||
/// `parent` is a function where `indexOf` is located.
|
||||
/// Example: `indexOf(arr, x) = 1`, parent is a function named `equals`.
|
||||
if (const auto * function = parent->as<ASTFunction>())
|
||||
{
|
||||
if (function->name == "and")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
else if (function->name == "equals" /// notEquals is not applicable
|
||||
|| function->name == "greater" || function->name == "greaterOrEquals"
|
||||
|| function->name == "less" || function->name == "lessOrEquals")
|
||||
{
|
||||
if (function->arguments->children.size() != 2)
|
||||
return false;
|
||||
|
||||
/// We don't allow constant expressions like `indexOf(arr, x) = 1 + 0` but it's neglible.
|
||||
|
||||
/// We should return true when the corresponding expression implies that the array contains the element.
|
||||
/// Example: when `indexOf(arr, x)` > 10 is written, it means that arr definitely should contain the element
|
||||
/// (at least at 11th position but it does not matter).
|
||||
|
||||
bool reversed = false;
|
||||
const ASTLiteral * constant = nullptr;
|
||||
|
||||
if (const ASTLiteral * left = function->arguments->children[0]->as<ASTLiteral>())
|
||||
{
|
||||
constant = left;
|
||||
reversed = true;
|
||||
}
|
||||
else if (const ASTLiteral * right = function->arguments->children[1]->as<ASTLiteral>())
|
||||
{
|
||||
constant = right;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
|
||||
Field zero(0);
|
||||
return (function->name == "equals" /// indexOf(...) = c, c != 0
|
||||
&& !applyVisitor(FieldVisitorAccurateEquals(), constant->value, zero))
|
||||
|| (function->name == "notEquals" /// indexOf(...) != c, c = 0
|
||||
&& applyVisitor(FieldVisitorAccurateEquals(), constant->value, zero))
|
||||
|| (function->name == (reversed ? "less" : "greater") /// indexOf(...) > c, c >= 0
|
||||
&& !applyVisitor(FieldVisitorAccurateLess(), constant->value, zero))
|
||||
|| (function->name == (reversed ? "lessOrEquals" : "greaterOrEquals") /// indexOf(...) >= c, c > 0
|
||||
&& applyVisitor(FieldVisitorAccurateLess(), zero, constant->value));
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
bool MergeTreeIndexConditionBloomFilter::traverseASTEquals(
|
||||
const String & function_name, const ASTPtr & key_ast, const DataTypePtr & value_type, const Field & value_field, RPNElement & out)
|
||||
const String & function_name, const ASTPtr & key_ast, const DataTypePtr & value_type, const Field & value_field, RPNElement & out, const ASTPtr & parent)
|
||||
{
|
||||
if (header.has(key_ast->getColumnName()))
|
||||
{
|
||||
@ -311,21 +391,26 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTEquals(
|
||||
const DataTypePtr & index_type = header.getByPosition(position).type;
|
||||
const auto * array_type = typeid_cast<const DataTypeArray *>(index_type.get());
|
||||
|
||||
if (function_name == "has")
|
||||
if (function_name == "has" || function_name == "indexOf")
|
||||
{
|
||||
out.function = RPNElement::FUNCTION_HAS;
|
||||
|
||||
if (!array_type)
|
||||
throw Exception("First argument for function has must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
throw Exception("First argument for function " + function_name + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
const DataTypePtr actual_type = BloomFilter::getPrimitiveType(array_type->getNestedType());
|
||||
Field converted_field = convertFieldToType(value_field, *actual_type, value_type.get());
|
||||
out.predicate.emplace_back(std::make_pair(position, BloomFilterHash::hashWithField(actual_type.get(), converted_field)));
|
||||
/// We can treat `indexOf` function similar to `has`.
|
||||
/// But it is little more cumbersome, compare: `has(arr, elem)` and `indexOf(arr, elem) != 0`.
|
||||
/// The `parent` in this context is expected to be function `!=` (`notEquals`).
|
||||
if (function_name == "has" || indexOfCanUseBloomFilter(parent))
|
||||
{
|
||||
out.function = RPNElement::FUNCTION_HAS;
|
||||
const DataTypePtr actual_type = BloomFilter::getPrimitiveType(array_type->getNestedType());
|
||||
Field converted_field = convertFieldToType(value_field, *actual_type, value_type.get());
|
||||
out.predicate.emplace_back(std::make_pair(position, BloomFilterHash::hashWithField(actual_type.get(), converted_field)));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (array_type)
|
||||
throw Exception("An array type of bloom_filter supports only has() function.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
throw Exception("An array type of bloom_filter supports only has() and indexOf() function.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
out.function = function_name == "equals" ? RPNElement::FUNCTION_EQUALS : RPNElement::FUNCTION_NOT_EQUALS;
|
||||
const DataTypePtr actual_type = BloomFilter::getPrimitiveType(index_type);
|
||||
@ -353,7 +438,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTEquals(
|
||||
const DataTypes & subtypes = value_tuple_data_type->getElements();
|
||||
|
||||
for (size_t index = 0; index < tuple.size(); ++index)
|
||||
match_with_subtype |= traverseASTEquals(function_name, arguments[index], subtypes[index], tuple[index], out);
|
||||
match_with_subtype |= traverseASTEquals(function_name, arguments[index], subtypes[index], tuple[index], out, key_ast);
|
||||
|
||||
return match_with_subtype;
|
||||
}
|
||||
|
@ -67,13 +67,15 @@ private:
|
||||
|
||||
bool traverseAtomAST(const ASTPtr & node, Block & block_with_constants, RPNElement & out);
|
||||
|
||||
bool traverseFunction(const ASTPtr & node, Block & block_with_constants, RPNElement & out, const ASTPtr & parent);
|
||||
|
||||
bool traverseASTIn(const String & function_name, const ASTPtr & key_ast, const SetPtr & prepared_set, RPNElement & out);
|
||||
|
||||
bool traverseASTIn(
|
||||
const String & function_name, const ASTPtr & key_ast, const DataTypePtr & type, const ColumnPtr & column, RPNElement & out);
|
||||
|
||||
bool traverseASTEquals(
|
||||
const String & function_name, const ASTPtr & key_ast, const DataTypePtr & value_type, const Field & value_field, RPNElement & out);
|
||||
const String & function_name, const ASTPtr & key_ast, const DataTypePtr & value_type, const Field & value_field, RPNElement & out, const ASTPtr & parent);
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -35,9 +35,10 @@ struct ReplicatedMergeTreeMutationEntry
|
||||
/// Replica which initiated mutation
|
||||
String source_replica;
|
||||
|
||||
/// Accured numbers of blocks
|
||||
/// Acquired block numbers
|
||||
/// partition_id -> block_number
|
||||
std::map<String, Int64> block_numbers;
|
||||
using BlockNumbersType = std::map<String, Int64>;
|
||||
BlockNumbersType block_numbers;
|
||||
|
||||
/// Mutation commands which will give to MUTATE_PART entries
|
||||
MutationCommands commands;
|
||||
|
@ -45,6 +45,16 @@ public:
|
||||
return part->storage.getVirtuals();
|
||||
}
|
||||
|
||||
String getPartitionId() const
|
||||
{
|
||||
return part->info.partition_id;
|
||||
}
|
||||
|
||||
String getPartitionIDFromQuery(const ASTPtr & ast, const Context & context) const
|
||||
{
|
||||
return part->storage.getPartitionIDFromQuery(ast, context);
|
||||
}
|
||||
|
||||
protected:
|
||||
StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_)
|
||||
: IStorage(getIDFromPart(part_))
|
||||
|
@ -2,11 +2,13 @@
|
||||
#include <IO/Operators.h>
|
||||
#include <Parsers/formatAST.h>
|
||||
#include <Parsers/ExpressionListParsers.h>
|
||||
#include <Parsers/ASTColumnDeclaration.h>
|
||||
#include <Parsers/ParserAlterQuery.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <Parsers/ASTAssignment.h>
|
||||
#include <Parsers/ASTColumnDeclaration.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <Core/Defines.h>
|
||||
@ -32,6 +34,7 @@ std::optional<MutationCommand> MutationCommand::parse(ASTAlterCommand * command,
|
||||
res.ast = command->ptr();
|
||||
res.type = DELETE;
|
||||
res.predicate = command->predicate;
|
||||
res.partition = command->partition;
|
||||
return res;
|
||||
}
|
||||
else if (command->type == ASTAlterCommand::UPDATE)
|
||||
@ -40,6 +43,7 @@ std::optional<MutationCommand> MutationCommand::parse(ASTAlterCommand * command,
|
||||
res.ast = command->ptr();
|
||||
res.type = UPDATE;
|
||||
res.predicate = command->predicate;
|
||||
res.partition = command->partition;
|
||||
for (const ASTPtr & assignment_ast : command->update_assignments->children)
|
||||
{
|
||||
const auto & assignment = assignment_ast->as<ASTAssignment &>();
|
||||
@ -124,6 +128,7 @@ std::shared_ptr<ASTAlterCommandList> MutationCommands::ast() const
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
void MutationCommands::writeText(WriteBuffer & out) const
|
||||
{
|
||||
std::stringstream commands_ss;
|
||||
|
@ -43,8 +43,10 @@ struct MutationCommand
|
||||
/// Columns with corresponding actions
|
||||
std::unordered_map<String, ASTPtr> column_to_update_expression;
|
||||
|
||||
/// For MATERIALIZE INDEX
|
||||
/// For MATERIALIZE INDEX.
|
||||
String index_name;
|
||||
|
||||
/// For MATERIALIZE INDEX, UPDATE and DELETE.
|
||||
ASTPtr partition;
|
||||
|
||||
/// For reads, drops and etc.
|
||||
|
@ -21,6 +21,7 @@ std::optional<PartitionCommand> PartitionCommand::parse(const ASTAlterCommand *
|
||||
res.type = DROP_PARTITION;
|
||||
res.partition = command_ast->partition;
|
||||
res.detach = command_ast->detach;
|
||||
res.part = command_ast->part;
|
||||
return res;
|
||||
}
|
||||
else if (command_ast->type == ASTAlterCommand::DROP_DETACHED_PARTITION)
|
||||
|
@ -42,13 +42,16 @@ void RabbitMQBlockOutputStream::writePrefix()
|
||||
|
||||
buffer->activateWriting();
|
||||
|
||||
child = FormatFactory::instance().getOutput(
|
||||
storage.getFormatName(), *buffer, getHeader(), context, [this](const Columns & /* columns */, size_t /* rows */)
|
||||
{
|
||||
buffer->countRow();
|
||||
},
|
||||
/* ignore_no_row_delimiter = */ true
|
||||
);
|
||||
auto format_settings = getFormatSettings(context);
|
||||
format_settings.protobuf.allow_many_rows_no_delimiters = true;
|
||||
|
||||
child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer,
|
||||
getHeader(), context,
|
||||
[this](const Columns & /* columns */, size_t /* rows */)
|
||||
{
|
||||
buffer->countRow();
|
||||
},
|
||||
format_settings);
|
||||
}
|
||||
|
||||
|
||||
|
@ -4,6 +4,7 @@
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/evaluateConstantExpression.h>
|
||||
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
|
||||
@ -202,6 +203,7 @@ StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArgu
|
||||
StorageFile::StorageFile(CommonArguments args)
|
||||
: IStorage(args.table_id)
|
||||
, format_name(args.format_name)
|
||||
, format_settings(args.format_settings)
|
||||
, compression_method(args.compression_method)
|
||||
, base_path(args.context.getPath())
|
||||
{
|
||||
@ -324,9 +326,11 @@ public:
|
||||
method = chooseCompressionMethod(current_path, storage->compression_method);
|
||||
}
|
||||
|
||||
read_buf = wrapReadBufferWithCompressionMethod(std::move(nested_buffer), method);
|
||||
reader = FormatFactory::instance().getInput(
|
||||
storage->format_name, *read_buf, metadata_snapshot->getSampleBlock(), context, max_block_size);
|
||||
read_buf = wrapReadBufferWithCompressionMethod(
|
||||
std::move(nested_buffer), method);
|
||||
reader = FormatFactory::instance().getInput(storage->format_name,
|
||||
*read_buf, metadata_snapshot->getSampleBlock(), context,
|
||||
max_block_size, storage->format_settings);
|
||||
|
||||
if (columns_description.hasDefaults())
|
||||
reader = std::make_shared<AddingDefaultsBlockInputStream>(reader, columns_description, context);
|
||||
@ -430,8 +434,11 @@ Pipe StorageFile::read(
|
||||
pipes.reserve(num_streams);
|
||||
|
||||
for (size_t i = 0; i < num_streams; ++i)
|
||||
{
|
||||
pipes.emplace_back(std::make_shared<StorageFileSource>(
|
||||
this_ptr, metadata_snapshot, context, max_block_size, files_info, metadata_snapshot->getColumns()));
|
||||
this_ptr, metadata_snapshot, context, max_block_size, files_info,
|
||||
metadata_snapshot->getColumns()));
|
||||
}
|
||||
|
||||
return Pipe::unitePipes(std::move(pipes));
|
||||
}
|
||||
@ -444,7 +451,8 @@ public:
|
||||
StorageFile & storage_,
|
||||
const StorageMetadataPtr & metadata_snapshot_,
|
||||
const CompressionMethod compression_method,
|
||||
const Context & context)
|
||||
const Context & context,
|
||||
const std::optional<FormatSettings> & format_settings)
|
||||
: storage(storage_)
|
||||
, metadata_snapshot(metadata_snapshot_)
|
||||
, lock(storage.rwlock)
|
||||
@ -472,7 +480,9 @@ public:
|
||||
|
||||
write_buf = wrapWriteBufferWithCompressionMethod(std::move(naked_buffer), compression_method, 3);
|
||||
|
||||
writer = FormatFactory::instance().getOutput(storage.format_name, *write_buf, metadata_snapshot->getSampleBlock(), context);
|
||||
writer = FormatFactory::instance().getOutput(storage.format_name,
|
||||
*write_buf, metadata_snapshot->getSampleBlock(), context,
|
||||
{}, format_settings);
|
||||
}
|
||||
|
||||
Block getHeader() const override { return metadata_snapshot->getSampleBlock(); }
|
||||
@ -521,7 +531,8 @@ BlockOutputStreamPtr StorageFile::write(
|
||||
path = paths[0];
|
||||
|
||||
return std::make_shared<StorageFileBlockOutputStream>(*this, metadata_snapshot,
|
||||
chooseCompressionMethod(path, compression_method), context);
|
||||
chooseCompressionMethod(path, compression_method), context,
|
||||
format_settings);
|
||||
}
|
||||
|
||||
bool StorageFile::storesDataOnDisk() const
|
||||
@ -586,32 +597,71 @@ void StorageFile::truncate(
|
||||
|
||||
void registerStorageFile(StorageFactory & factory)
|
||||
{
|
||||
StorageFactory::StorageFeatures storage_features{
|
||||
.supports_settings = true,
|
||||
.source_access_type = AccessType::FILE
|
||||
};
|
||||
|
||||
factory.registerStorage(
|
||||
"File",
|
||||
[](const StorageFactory::Arguments & args)
|
||||
[](const StorageFactory::Arguments & factory_args)
|
||||
{
|
||||
ASTs & engine_args = args.engine_args;
|
||||
StorageFile::CommonArguments storage_args{
|
||||
.table_id = factory_args.table_id,
|
||||
.columns = factory_args.columns,
|
||||
.constraints = factory_args.constraints,
|
||||
.context = factory_args.context
|
||||
};
|
||||
|
||||
if (!(engine_args.size() >= 1 && engine_args.size() <= 3)) // NOLINT
|
||||
ASTs & engine_args_ast = factory_args.engine_args;
|
||||
|
||||
if (!(engine_args_ast.size() >= 1 && engine_args_ast.size() <= 3)) // NOLINT
|
||||
throw Exception(
|
||||
"Storage File requires from 1 to 3 arguments: name of used format, source and compression_method.",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context);
|
||||
String format_name = engine_args[0]->as<ASTLiteral &>().value.safeGet<String>();
|
||||
engine_args_ast[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[0], factory_args.local_context);
|
||||
storage_args.format_name = engine_args_ast[0]->as<ASTLiteral &>().value.safeGet<String>();
|
||||
|
||||
String compression_method;
|
||||
StorageFile::CommonArguments common_args{
|
||||
args.table_id, format_name, compression_method, args.columns, args.constraints, args.context};
|
||||
// Use format settings from global server context + settings from
|
||||
// the SETTINGS clause of the create query. Settings from current
|
||||
// session and user are ignored.
|
||||
if (factory_args.storage_def->settings)
|
||||
{
|
||||
FormatFactorySettings user_format_settings;
|
||||
|
||||
if (engine_args.size() == 1) /// Table in database
|
||||
return StorageFile::create(args.relative_data_path, common_args);
|
||||
// Apply changed settings from global context, but ignore the
|
||||
// unknown ones, because we only have the format settings here.
|
||||
const auto & changes = factory_args.context.getSettingsRef().changes();
|
||||
for (const auto & change : changes)
|
||||
{
|
||||
if (user_format_settings.has(change.name))
|
||||
{
|
||||
user_format_settings.set(change.name, change.value);
|
||||
}
|
||||
}
|
||||
|
||||
// Apply changes from SETTINGS clause, with validation.
|
||||
user_format_settings.applyChanges(
|
||||
factory_args.storage_def->settings->changes);
|
||||
|
||||
storage_args.format_settings = getFormatSettings(
|
||||
factory_args.context, user_format_settings);
|
||||
}
|
||||
else
|
||||
{
|
||||
storage_args.format_settings = getFormatSettings(
|
||||
factory_args.context);
|
||||
}
|
||||
|
||||
if (engine_args_ast.size() == 1) /// Table in database
|
||||
return StorageFile::create(factory_args.relative_data_path, storage_args);
|
||||
|
||||
/// Will use FD if engine_args[1] is int literal or identifier with std* name
|
||||
int source_fd = -1;
|
||||
String source_path;
|
||||
|
||||
if (auto opt_name = tryGetIdentifierName(engine_args[1]))
|
||||
if (auto opt_name = tryGetIdentifierName(engine_args_ast[1]))
|
||||
{
|
||||
if (*opt_name == "stdin")
|
||||
source_fd = STDIN_FILENO;
|
||||
@ -623,7 +673,7 @@ void registerStorageFile(StorageFactory & factory)
|
||||
throw Exception(
|
||||
"Unknown identifier '" + *opt_name + "' in second arg of File storage constructor", ErrorCodes::UNKNOWN_IDENTIFIER);
|
||||
}
|
||||
else if (const auto * literal = engine_args[1]->as<ASTLiteral>())
|
||||
else if (const auto * literal = engine_args_ast[1]->as<ASTLiteral>())
|
||||
{
|
||||
auto type = literal->value.getType();
|
||||
if (type == Field::Types::Int64)
|
||||
@ -636,23 +686,23 @@ void registerStorageFile(StorageFactory & factory)
|
||||
throw Exception("Second argument must be path or file descriptor", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
if (engine_args.size() == 3)
|
||||
if (engine_args_ast.size() == 3)
|
||||
{
|
||||
engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], args.local_context);
|
||||
compression_method = engine_args[2]->as<ASTLiteral &>().value.safeGet<String>();
|
||||
engine_args_ast[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[2], factory_args.local_context);
|
||||
storage_args.compression_method = engine_args_ast[2]->as<ASTLiteral &>().value.safeGet<String>();
|
||||
}
|
||||
else
|
||||
compression_method = "auto";
|
||||
storage_args.compression_method = "auto";
|
||||
|
||||
if (0 <= source_fd) /// File descriptor
|
||||
return StorageFile::create(source_fd, common_args);
|
||||
return StorageFile::create(source_fd, storage_args);
|
||||
else /// User's file
|
||||
return StorageFile::create(source_path, args.context.getUserFilesPath(), common_args);
|
||||
return StorageFile::create(source_path, factory_args.context.getUserFilesPath(), storage_args);
|
||||
},
|
||||
{
|
||||
.source_access_type = AccessType::FILE,
|
||||
});
|
||||
storage_features);
|
||||
}
|
||||
|
||||
|
||||
NamesAndTypesList StorageFile::getVirtuals() const
|
||||
{
|
||||
return NamesAndTypesList{
|
||||
|
@ -51,9 +51,10 @@ public:
|
||||
|
||||
struct CommonArguments
|
||||
{
|
||||
const StorageID & table_id;
|
||||
const std::string & format_name;
|
||||
const std::string & compression_method;
|
||||
StorageID table_id;
|
||||
std::string format_name;
|
||||
std::optional<FormatSettings> format_settings;
|
||||
std::string compression_method;
|
||||
const ColumnsDescription & columns;
|
||||
const ConstraintsDescription & constraints;
|
||||
const Context & context;
|
||||
@ -80,6 +81,11 @@ private:
|
||||
explicit StorageFile(CommonArguments args);
|
||||
|
||||
std::string format_name;
|
||||
// We use format settings from global context + CREATE query for File table
|
||||
// function -- in this case, format_settings is set.
|
||||
// For `file` table function, we use format settings from current user context,
|
||||
// in this case, format_settings is not set.
|
||||
std::optional<FormatSettings> format_settings;
|
||||
|
||||
int table_fd = -1;
|
||||
String compression_method;
|
||||
|
@ -256,10 +256,10 @@ void StorageMaterializedView::checkAlterIsPossible(const AlterCommands & command
|
||||
}
|
||||
|
||||
Pipe StorageMaterializedView::alterPartition(
|
||||
const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & context)
|
||||
const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & context)
|
||||
{
|
||||
checkStatementCanBeForwarded();
|
||||
return getTargetTable()->alterPartition(query, metadata_snapshot, commands, context);
|
||||
return getTargetTable()->alterPartition(metadata_snapshot, commands, context);
|
||||
}
|
||||
|
||||
void StorageMaterializedView::checkAlterPartitionIsPossible(
|
||||
|
@ -52,7 +52,7 @@ public:
|
||||
|
||||
void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const override;
|
||||
|
||||
Pipe alterPartition(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & context) override;
|
||||
Pipe alterPartition(const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & context) override;
|
||||
|
||||
void checkAlterPartitionIsPossible(const PartitionCommands & commands, const StorageMetadataPtr & metadata_snapshot, const Settings & settings) const override;
|
||||
|
||||
|
@ -46,6 +46,7 @@ namespace ErrorCodes
|
||||
extern const int CANNOT_ASSIGN_OPTIMIZE;
|
||||
extern const int TIMEOUT_EXCEEDED;
|
||||
extern const int UNKNOWN_POLICY;
|
||||
extern const int NO_SUCH_DATA_PART;
|
||||
}
|
||||
|
||||
namespace ActionLocks
|
||||
@ -1056,7 +1057,6 @@ bool StorageMergeTree::optimize(
|
||||
}
|
||||
|
||||
Pipe StorageMergeTree::alterPartition(
|
||||
const ASTPtr & query,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const PartitionCommands & commands,
|
||||
const Context & query_context)
|
||||
@ -1068,8 +1068,11 @@ Pipe StorageMergeTree::alterPartition(
|
||||
switch (command.type)
|
||||
{
|
||||
case PartitionCommand::DROP_PARTITION:
|
||||
checkPartitionCanBeDropped(command.partition);
|
||||
dropPartition(command.partition, command.detach, query_context);
|
||||
if (command.part)
|
||||
checkPartCanBeDropped(command.partition);
|
||||
else
|
||||
checkPartitionCanBeDropped(command.partition);
|
||||
dropPartition(command.partition, command.detach, command.part, query_context);
|
||||
break;
|
||||
|
||||
case PartitionCommand::DROP_DETACHED_PARTITION:
|
||||
@ -1127,7 +1130,7 @@ Pipe StorageMergeTree::alterPartition(
|
||||
break;
|
||||
|
||||
default:
|
||||
IStorage::alterPartition(query, metadata_snapshot, commands, query_context); // should throw an exception.
|
||||
IStorage::alterPartition(metadata_snapshot, commands, query_context); // should throw an exception.
|
||||
}
|
||||
|
||||
for (auto & command_result : current_command_results)
|
||||
@ -1167,7 +1170,7 @@ ActionLock StorageMergeTree::stopMergesAndWait()
|
||||
}
|
||||
|
||||
|
||||
void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, const Context & context)
|
||||
void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool drop_part, const Context & context)
|
||||
{
|
||||
{
|
||||
/// Asks to complete merges and does not allow them to start.
|
||||
@ -1175,10 +1178,25 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons
|
||||
auto merge_blocker = stopMergesAndWait();
|
||||
|
||||
auto metadata_snapshot = getInMemoryMetadataPtr();
|
||||
String partition_id = getPartitionIDFromQuery(partition, context);
|
||||
|
||||
/// TODO: should we include PreComitted parts like in Replicated case?
|
||||
auto parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id);
|
||||
MergeTreeData::DataPartsVector parts_to_remove;
|
||||
|
||||
if (drop_part)
|
||||
{
|
||||
String part_name = partition->as<ASTLiteral &>().value.safeGet<String>();
|
||||
auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Committed});
|
||||
|
||||
if (part)
|
||||
parts_to_remove.push_back(part);
|
||||
else
|
||||
throw Exception("Part " + part_name + " not found, won't try to drop it.", ErrorCodes::NO_SUCH_DATA_PART);
|
||||
}
|
||||
else
|
||||
{
|
||||
String partition_id = getPartitionIDFromQuery(partition, context);
|
||||
parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id);
|
||||
}
|
||||
|
||||
// TODO should we throw an exception if parts_to_remove is empty?
|
||||
removePartsFromWorkingSet(parts_to_remove, true);
|
||||
|
||||
@ -1193,9 +1211,9 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons
|
||||
}
|
||||
|
||||
if (detach)
|
||||
LOG_INFO(log, "Detached {} parts inside partition ID {}.", parts_to_remove.size(), partition_id);
|
||||
LOG_INFO(log, "Detached {} parts.", parts_to_remove.size());
|
||||
else
|
||||
LOG_INFO(log, "Removed {} parts inside partition ID {}.", parts_to_remove.size(), partition_id);
|
||||
LOG_INFO(log, "Removed {} parts.", parts_to_remove.size());
|
||||
}
|
||||
|
||||
clearOldPartsFromFilesystem();
|
||||
|
@ -63,7 +63,6 @@ public:
|
||||
const Context & context) override;
|
||||
|
||||
Pipe alterPartition(
|
||||
const ASTPtr & query,
|
||||
const StorageMetadataPtr & /* metadata_snapshot */,
|
||||
const PartitionCommands & commands,
|
||||
const Context & context) override;
|
||||
@ -183,7 +182,7 @@ private:
|
||||
void clearOldMutations(bool truncate = false);
|
||||
|
||||
// Partition helpers
|
||||
void dropPartition(const ASTPtr & partition, bool detach, const Context & context);
|
||||
void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, const Context & context);
|
||||
PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, bool part, const Context & context);
|
||||
|
||||
void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context);
|
||||
|
@ -104,12 +104,11 @@ public:
|
||||
}
|
||||
|
||||
Pipe alterPartition(
|
||||
const ASTPtr & query,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const PartitionCommands & commands,
|
||||
const Context & context) override
|
||||
{
|
||||
return getNested()->alterPartition(query, metadata_snapshot, commands, context);
|
||||
return getNested()->alterPartition(metadata_snapshot, commands, context);
|
||||
}
|
||||
|
||||
void checkAlterPartitionIsPossible(const PartitionCommands & commands, const StorageMetadataPtr & metadata_snapshot, const Settings & settings) const override
|
||||
|
@ -120,6 +120,7 @@ namespace ErrorCodes
|
||||
extern const int DIRECTORY_ALREADY_EXISTS;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int UNKNOWN_POLICY;
|
||||
extern const int NO_SUCH_DATA_PART;
|
||||
}
|
||||
|
||||
namespace ActionLocks
|
||||
@ -3301,6 +3302,33 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id)
|
||||
}
|
||||
|
||||
|
||||
bool StorageReplicatedMergeTree::partIsInsertingWithParallelQuorum(const MergeTreePartInfo & part_info) const
|
||||
{
|
||||
auto zookeeper = getZooKeeper();
|
||||
return zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_info.getPartName());
|
||||
}
|
||||
|
||||
bool StorageReplicatedMergeTree::partIsLastQuorumPart(const MergeTreePartInfo & part_info) const
|
||||
{
|
||||
auto zookeeper = getZooKeeper();
|
||||
|
||||
const String parts_with_quorum_path = zookeeper_path + "/quorum/last_part";
|
||||
|
||||
String parts_with_quorum_str = zookeeper->get(parts_with_quorum_path);
|
||||
|
||||
if (parts_with_quorum_str.empty())
|
||||
return false;
|
||||
|
||||
ReplicatedMergeTreeQuorumAddedParts parts_with_quorum(format_version);
|
||||
parts_with_quorum.fromString(parts_with_quorum_str);
|
||||
|
||||
auto partition_it = parts_with_quorum.added_parts.find(part_info.partition_id);
|
||||
if (partition_it == parts_with_quorum.added_parts.end())
|
||||
return false;
|
||||
|
||||
return partition_it->second == part_info.getPartName();
|
||||
}
|
||||
|
||||
bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const StorageMetadataPtr & metadata_snapshot,
|
||||
const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_)
|
||||
{
|
||||
@ -3919,6 +3947,60 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer
|
||||
}
|
||||
|
||||
|
||||
std::set<String> StorageReplicatedMergeTree::getPartitionIdsAffectedByCommands(
|
||||
const MutationCommands & commands, const Context & query_context) const
|
||||
{
|
||||
std::set<String> affected_partition_ids;
|
||||
|
||||
for (const auto & command : commands)
|
||||
{
|
||||
if (!command.partition)
|
||||
{
|
||||
affected_partition_ids.clear();
|
||||
break;
|
||||
}
|
||||
|
||||
affected_partition_ids.insert(
|
||||
getPartitionIDFromQuery(command.partition, query_context)
|
||||
);
|
||||
}
|
||||
|
||||
return affected_partition_ids;
|
||||
}
|
||||
|
||||
|
||||
PartitionBlockNumbersHolder StorageReplicatedMergeTree::allocateBlockNumbersInAffectedPartitions(
|
||||
const MutationCommands & commands, const Context & query_context, const zkutil::ZooKeeperPtr & zookeeper) const
|
||||
{
|
||||
const std::set<String> mutation_affected_partition_ids = getPartitionIdsAffectedByCommands(commands, query_context);
|
||||
|
||||
if (mutation_affected_partition_ids.size() == 1)
|
||||
{
|
||||
const auto & affected_partition_id = *mutation_affected_partition_ids.cbegin();
|
||||
auto block_number_holder = allocateBlockNumber(affected_partition_id, zookeeper);
|
||||
if (!block_number_holder.has_value())
|
||||
return {};
|
||||
auto block_number = block_number_holder->getNumber(); /// Avoid possible UB due to std::move
|
||||
return {{{affected_partition_id, block_number}}, std::move(block_number_holder)};
|
||||
}
|
||||
else
|
||||
{
|
||||
/// TODO: Implement optimal block number aqcuisition algorithm in multiple (but not all) partitions
|
||||
EphemeralLocksInAllPartitions lock_holder(
|
||||
zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper);
|
||||
|
||||
PartitionBlockNumbersHolder::BlockNumbersType block_numbers;
|
||||
for (const auto & lock : lock_holder.getLocks())
|
||||
{
|
||||
if (mutation_affected_partition_ids.empty() || mutation_affected_partition_ids.count(lock.partition_id))
|
||||
block_numbers[lock.partition_id] = lock.number;
|
||||
}
|
||||
|
||||
return {std::move(block_numbers), std::move(lock_holder)};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void StorageReplicatedMergeTree::alter(
|
||||
const AlterCommands & commands, const Context & query_context, TableLockHolder & table_lock_holder)
|
||||
{
|
||||
@ -3946,7 +4028,7 @@ void StorageReplicatedMergeTree::alter(
|
||||
return queryToString(query);
|
||||
};
|
||||
|
||||
auto zookeeper = getZooKeeper();
|
||||
const auto zookeeper = getZooKeeper();
|
||||
|
||||
std::optional<ReplicatedMergeTreeLogEntryData> alter_entry;
|
||||
std::optional<String> mutation_znode;
|
||||
@ -3957,10 +4039,6 @@ void StorageReplicatedMergeTree::alter(
|
||||
alter_entry.emplace();
|
||||
mutation_znode.reset();
|
||||
|
||||
/// We can safely read structure, because we guarded with alter_intention_lock
|
||||
if (is_readonly)
|
||||
throw Exception("Can't ALTER readonly table", ErrorCodes::TABLE_IS_READ_ONLY);
|
||||
|
||||
auto current_metadata = getInMemoryMetadataPtr();
|
||||
|
||||
StorageInMemoryMetadata future_metadata = *current_metadata;
|
||||
@ -4033,27 +4111,23 @@ void StorageReplicatedMergeTree::alter(
|
||||
ops.emplace_back(zkutil::makeCreateRequest(
|
||||
zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential));
|
||||
|
||||
std::optional<EphemeralLocksInAllPartitions> lock_holder;
|
||||
|
||||
/// Now we will prepare mutations record.
|
||||
/// This code pretty same with mutate() function but process results slightly differently.
|
||||
PartitionBlockNumbersHolder partition_block_numbers_holder;
|
||||
if (alter_entry->have_mutation)
|
||||
{
|
||||
String mutations_path = zookeeper_path + "/mutations";
|
||||
const String mutations_path(zookeeper_path + "/mutations");
|
||||
|
||||
ReplicatedMergeTreeMutationEntry mutation_entry;
|
||||
mutation_entry.source_replica = replica_name;
|
||||
mutation_entry.commands = maybe_mutation_commands;
|
||||
mutation_entry.alter_version = new_metadata_version;
|
||||
mutation_entry.source_replica = replica_name;
|
||||
mutation_entry.commands = std::move(maybe_mutation_commands);
|
||||
|
||||
Coordination::Stat mutations_stat;
|
||||
zookeeper->get(mutations_path, &mutations_stat);
|
||||
|
||||
lock_holder.emplace(
|
||||
zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper);
|
||||
|
||||
for (const auto & lock : lock_holder->getLocks())
|
||||
mutation_entry.block_numbers[lock.partition_id] = lock.number;
|
||||
partition_block_numbers_holder =
|
||||
allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper);
|
||||
|
||||
mutation_entry.block_numbers = partition_block_numbers_holder.getBlockNumbers();
|
||||
mutation_entry.create_time = time(nullptr);
|
||||
|
||||
ops.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version));
|
||||
@ -4064,6 +4138,11 @@ void StorageReplicatedMergeTree::alter(
|
||||
Coordination::Responses results;
|
||||
Coordination::Error rc = zookeeper->tryMulti(ops, results);
|
||||
|
||||
/// For the sake of constitency with mechanics of concurrent background process of assigning parts merge tasks
|
||||
/// this placeholder must be held up until the moment of committing into ZK of the mutation entry
|
||||
/// See ReplicatedMergeTreeMergePredicate::canMergeTwoParts() method
|
||||
partition_block_numbers_holder.reset();
|
||||
|
||||
if (rc == Coordination::Error::ZOK)
|
||||
{
|
||||
if (alter_entry->have_mutation)
|
||||
@ -4124,7 +4203,6 @@ void StorageReplicatedMergeTree::alter(
|
||||
}
|
||||
|
||||
Pipe StorageReplicatedMergeTree::alterPartition(
|
||||
const ASTPtr & query,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const PartitionCommands & commands,
|
||||
const Context & query_context)
|
||||
@ -4136,8 +4214,11 @@ Pipe StorageReplicatedMergeTree::alterPartition(
|
||||
switch (command.type)
|
||||
{
|
||||
case PartitionCommand::DROP_PARTITION:
|
||||
checkPartitionCanBeDropped(command.partition);
|
||||
dropPartition(query, command.partition, command.detach, query_context);
|
||||
if (command.part)
|
||||
checkPartCanBeDropped(command.partition);
|
||||
else
|
||||
checkPartitionCanBeDropped(command.partition);
|
||||
dropPartition(command.partition, command.detach, command.part, query_context);
|
||||
break;
|
||||
|
||||
case PartitionCommand::DROP_DETACHED_PARTITION:
|
||||
@ -4267,18 +4348,29 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition(const St
|
||||
}
|
||||
|
||||
|
||||
void StorageReplicatedMergeTree::dropPartition(const ASTPtr &, const ASTPtr & partition, bool detach, const Context & query_context)
|
||||
void StorageReplicatedMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool drop_part, const Context & query_context)
|
||||
{
|
||||
assertNotReadonly();
|
||||
if (!is_leader)
|
||||
throw Exception("DROP PARTITION cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER);
|
||||
throw Exception("DROP PART|PARTITION cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER);
|
||||
|
||||
zkutil::ZooKeeperPtr zookeeper = getZooKeeper();
|
||||
|
||||
String partition_id = getPartitionIDFromQuery(partition, query_context);
|
||||
|
||||
LogEntry entry;
|
||||
if (dropPartsInPartition(*zookeeper, partition_id, entry, detach))
|
||||
bool did_drop;
|
||||
|
||||
if (drop_part)
|
||||
{
|
||||
String part_name = partition->as<ASTLiteral &>().value.safeGet<String>();
|
||||
did_drop = dropPart(zookeeper, part_name, entry, detach);
|
||||
}
|
||||
else
|
||||
{
|
||||
String partition_id = getPartitionIDFromQuery(partition, query_context);
|
||||
did_drop = dropAllPartsInPartition(*zookeeper, partition_id, entry, detach);
|
||||
}
|
||||
|
||||
if (did_drop)
|
||||
{
|
||||
/// If necessary, wait until the operation is performed on itself or on all replicas.
|
||||
if (query_context.getSettingsRef().replication_alter_partitions_sync != 0)
|
||||
@ -4290,8 +4382,11 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr &, const ASTPtr & pa
|
||||
}
|
||||
}
|
||||
|
||||
/// Cleaning possibly stored information about parts from /quorum/last_part node in ZooKeeper.
|
||||
cleanLastPartNode(partition_id);
|
||||
if (!drop_part)
|
||||
{
|
||||
String partition_id = getPartitionIDFromQuery(partition, query_context);
|
||||
cleanLastPartNode(partition_id);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -4312,7 +4407,7 @@ void StorageReplicatedMergeTree::truncate(
|
||||
{
|
||||
LogEntry entry;
|
||||
|
||||
if (dropPartsInPartition(*zookeeper, partition_id, entry, false))
|
||||
if (dropAllPartsInPartition(*zookeeper, partition_id, entry, false))
|
||||
waitForAllReplicasToProcessLogEntry(entry);
|
||||
}
|
||||
}
|
||||
@ -4386,7 +4481,7 @@ void StorageReplicatedMergeTree::rename(const String & new_path_to_table_data, c
|
||||
}
|
||||
|
||||
|
||||
bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path)
|
||||
bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) const
|
||||
{
|
||||
{
|
||||
std::lock_guard lock(existing_nodes_cache_mutex);
|
||||
@ -4408,7 +4503,7 @@ bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path)
|
||||
|
||||
std::optional<EphemeralLockInZooKeeper>
|
||||
StorageReplicatedMergeTree::allocateBlockNumber(
|
||||
const String & partition_id, zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_block_id_path)
|
||||
const String & partition_id, const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_block_id_path) const
|
||||
{
|
||||
/// Lets check for duplicates in advance, to avoid superfluous block numbers allocation
|
||||
Coordination::Requests deduplication_check_ops;
|
||||
@ -5051,44 +5146,46 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const
|
||||
/// After all needed parts are mutated (i.e. all active parts have the mutation version greater than
|
||||
/// the version of this mutation), the mutation is considered done and can be deleted.
|
||||
|
||||
ReplicatedMergeTreeMutationEntry entry;
|
||||
entry.source_replica = replica_name;
|
||||
entry.commands = commands;
|
||||
ReplicatedMergeTreeMutationEntry mutation_entry;
|
||||
mutation_entry.source_replica = replica_name;
|
||||
mutation_entry.commands = commands;
|
||||
|
||||
String mutations_path = zookeeper_path + "/mutations";
|
||||
const String mutations_path = zookeeper_path + "/mutations";
|
||||
const auto zookeeper = getZooKeeper();
|
||||
|
||||
/// Update the mutations_path node when creating the mutation and check its version to ensure that
|
||||
/// nodes for mutations are created in the same order as the corresponding block numbers.
|
||||
/// Should work well if the number of concurrent mutation requests is small.
|
||||
while (true)
|
||||
{
|
||||
auto zookeeper = getZooKeeper();
|
||||
|
||||
Coordination::Stat mutations_stat;
|
||||
zookeeper->get(mutations_path, &mutations_stat);
|
||||
|
||||
EphemeralLocksInAllPartitions block_number_locks(
|
||||
zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper);
|
||||
PartitionBlockNumbersHolder partition_block_numbers_holder =
|
||||
allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper);
|
||||
|
||||
for (const auto & lock : block_number_locks.getLocks())
|
||||
entry.block_numbers[lock.partition_id] = lock.number;
|
||||
|
||||
entry.create_time = time(nullptr);
|
||||
mutation_entry.block_numbers = partition_block_numbers_holder.getBlockNumbers();
|
||||
mutation_entry.create_time = time(nullptr);
|
||||
|
||||
/// The following version check guarantees the linearizability property for any pair of mutations:
|
||||
/// mutation with higher sequence number is guaranteed to have higher block numbers in every partition
|
||||
/// (and thus will be applied strictly according to sequence numbers of mutations)
|
||||
Coordination::Requests requests;
|
||||
requests.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version));
|
||||
requests.emplace_back(zkutil::makeCreateRequest(
|
||||
mutations_path + "/", entry.toString(), zkutil::CreateMode::PersistentSequential));
|
||||
mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential));
|
||||
|
||||
Coordination::Responses responses;
|
||||
Coordination::Error rc = zookeeper->tryMulti(requests, responses);
|
||||
|
||||
partition_block_numbers_holder.reset();
|
||||
|
||||
if (rc == Coordination::Error::ZOK)
|
||||
{
|
||||
const String & path_created =
|
||||
dynamic_cast<const Coordination::CreateResponse *>(responses[1].get())->path_created;
|
||||
entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1);
|
||||
LOG_TRACE(log, "Created mutation with ID {}", entry.znode_name);
|
||||
mutation_entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1);
|
||||
LOG_TRACE(log, "Created mutation with ID {}", mutation_entry.znode_name);
|
||||
break;
|
||||
}
|
||||
else if (rc == Coordination::Error::ZBADVERSION)
|
||||
@ -5100,7 +5197,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const
|
||||
throw Coordination::Exception("Unable to create a mutation znode", rc);
|
||||
}
|
||||
|
||||
waitMutation(entry.znode_name, query_context.getSettingsRef().mutations_sync);
|
||||
waitMutation(mutation_entry.znode_name, query_context.getSettingsRef().mutations_sync);
|
||||
}
|
||||
|
||||
void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t mutations_sync) const
|
||||
@ -5390,8 +5487,8 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper(
|
||||
}
|
||||
|
||||
|
||||
void StorageReplicatedMergeTree::clearBlocksInPartition(
|
||||
zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num)
|
||||
void StorageReplicatedMergeTree::getClearBlocksInPartitionOps(
|
||||
Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num)
|
||||
{
|
||||
Strings blocks;
|
||||
if (Coordination::Error::ZOK != zookeeper.tryGetChildren(zookeeper_path + "/blocks", blocks))
|
||||
@ -5408,7 +5505,6 @@ void StorageReplicatedMergeTree::clearBlocksInPartition(
|
||||
}
|
||||
}
|
||||
|
||||
zkutil::AsyncResponses<Coordination::RemoveResponse> to_delete_futures;
|
||||
for (auto & pair : get_futures)
|
||||
{
|
||||
const String & path = pair.first;
|
||||
@ -5421,23 +5517,25 @@ void StorageReplicatedMergeTree::clearBlocksInPartition(
|
||||
Int64 block_num = 0;
|
||||
bool parsed = tryReadIntText(block_num, buf) && buf.eof();
|
||||
if (!parsed || (min_block_num <= block_num && block_num <= max_block_num))
|
||||
to_delete_futures.emplace_back(path, zookeeper.asyncTryRemove(path));
|
||||
ops.emplace_back(zkutil::makeRemoveRequest(path, -1));
|
||||
}
|
||||
}
|
||||
|
||||
for (auto & pair : to_delete_futures)
|
||||
void StorageReplicatedMergeTree::clearBlocksInPartition(
|
||||
zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num)
|
||||
{
|
||||
Coordination::Requests delete_requests;
|
||||
getClearBlocksInPartitionOps(delete_requests, zookeeper, partition_id, min_block_num, max_block_num);
|
||||
Coordination::Responses delete_responses;
|
||||
auto code = zookeeper.tryMulti(delete_requests, delete_responses);
|
||||
if (code != Coordination::Error::ZOK)
|
||||
{
|
||||
const String & path = pair.first;
|
||||
Coordination::Error rc = pair.second.get().error;
|
||||
if (rc == Coordination::Error::ZNOTEMPTY)
|
||||
{
|
||||
/// Can happen if there are leftover block nodes with children created by previous server versions.
|
||||
zookeeper.removeRecursive(path);
|
||||
}
|
||||
else if (rc != Coordination::Error::ZOK)
|
||||
LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, Coordination::errorMessage(rc));
|
||||
for (size_t i = 0; i < delete_requests.size(); ++i)
|
||||
if (delete_responses[i]->error != Coordination::Error::ZOK)
|
||||
LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", delete_requests[i]->getPath(), Coordination::errorMessage(delete_responses[i]->error));
|
||||
}
|
||||
|
||||
LOG_TRACE(log, "Deleted {} deduplication block IDs in partition ID {}", to_delete_futures.size(), partition_id);
|
||||
LOG_TRACE(log, "Deleted {} deduplication block IDs in partition ID {}", delete_requests.size(), partition_id);
|
||||
}
|
||||
|
||||
void StorageReplicatedMergeTree::replacePartitionFrom(
|
||||
@ -5956,9 +6054,72 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI
|
||||
return true;
|
||||
}
|
||||
|
||||
bool StorageReplicatedMergeTree::dropPart(
|
||||
zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach)
|
||||
{
|
||||
LOG_TRACE(log, "Will try to insert a log entry to DROP_RANGE for part: " + part_name);
|
||||
|
||||
bool StorageReplicatedMergeTree::dropPartsInPartition(
|
||||
zkutil::ZooKeeper & zookeeper, String & partition_id, StorageReplicatedMergeTree::LogEntry & entry, bool detach)
|
||||
auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper);
|
||||
|
||||
auto part = getPartIfExists(part_info, {MergeTreeDataPartState::Committed});
|
||||
|
||||
if (!part)
|
||||
throw Exception("Part " + part_name + " not found locally, won't try to drop it.", ErrorCodes::NO_SUCH_DATA_PART);
|
||||
|
||||
/// There isn't a lot we can do otherwise. Can't cancel merges because it is possible that a replica already
|
||||
/// finished the merge.
|
||||
if (partIsAssignedToBackgroundOperation(part))
|
||||
throw Exception("Part " + part_name
|
||||
+ " is currently participating in a background operation (mutation/merge)"
|
||||
+ ", try again later", ErrorCodes::PART_IS_TEMPORARILY_LOCKED);
|
||||
|
||||
if (partIsLastQuorumPart(part->info))
|
||||
throw Exception("Part " + part_name + " is last inserted part with quorum in partition. Cannot drop",
|
||||
ErrorCodes::NOT_IMPLEMENTED);
|
||||
|
||||
if (partIsInsertingWithParallelQuorum(part->info))
|
||||
throw Exception("Part " + part_name + " is inserting with parallel quorum. Cannot drop",
|
||||
ErrorCodes::NOT_IMPLEMENTED);
|
||||
|
||||
Coordination::Requests ops;
|
||||
getClearBlocksInPartitionOps(ops, *zookeeper, part_info.partition_id, part_info.min_block, part_info.max_block);
|
||||
size_t clean_block_ops_size = ops.size();
|
||||
|
||||
/// If `part_name` is result of a recent merge and source parts are still available then
|
||||
/// DROP_RANGE with detach will move this part together with source parts to `detached/` dir.
|
||||
entry.type = LogEntry::DROP_RANGE;
|
||||
entry.source_replica = replica_name;
|
||||
entry.new_part_name = part_name;
|
||||
entry.detach = detach;
|
||||
entry.create_time = time(nullptr);
|
||||
|
||||
ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/log", merge_pred.getVersion())); /// Make sure no new events were added to the log.
|
||||
ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential));
|
||||
ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); /// Just update version.
|
||||
Coordination::Responses responses;
|
||||
Coordination::Error rc = zookeeper->tryMulti(ops, responses);
|
||||
|
||||
if (rc == Coordination::Error::ZBADVERSION)
|
||||
{
|
||||
LOG_TRACE(log, "A new log entry appeared while trying to commit DROP RANGE. Retry.");
|
||||
continue;
|
||||
}
|
||||
else
|
||||
zkutil::KeeperMultiException::check(rc, ops, responses);
|
||||
|
||||
String log_znode_path = dynamic_cast<const Coordination::CreateResponse &>(*responses[clean_block_ops_size + 1]).path_created;
|
||||
entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
bool StorageReplicatedMergeTree::dropAllPartsInPartition(
|
||||
zkutil::ZooKeeper & zookeeper, String & partition_id, LogEntry & entry, bool detach)
|
||||
{
|
||||
MergeTreePartInfo drop_range_info;
|
||||
if (!getFakePartCoveringAllPartsInPartition(partition_id, drop_range_info))
|
||||
|
@ -113,7 +113,6 @@ public:
|
||||
void alter(const AlterCommands & commands, const Context & query_context, TableLockHolder & table_lock_holder) override;
|
||||
|
||||
Pipe alterPartition(
|
||||
const ASTPtr & query,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const PartitionCommands & commands,
|
||||
const Context & query_context) override;
|
||||
@ -506,10 +505,16 @@ private:
|
||||
/// Deletes info from quorum/last_part node for particular partition_id.
|
||||
void cleanLastPartNode(const String & partition_id);
|
||||
|
||||
/// Part name is stored in quorum/last_part for corresponding partition_id.
|
||||
bool partIsLastQuorumPart(const MergeTreePartInfo & part_info) const;
|
||||
|
||||
/// Part currently inserting with quorum (node quorum/parallel/part_name exists)
|
||||
bool partIsInsertingWithParallelQuorum(const MergeTreePartInfo & part_info) const;
|
||||
|
||||
/// Creates new block number if block with such block_id does not exist
|
||||
std::optional<EphemeralLockInZooKeeper> allocateBlockNumber(
|
||||
const String & partition_id, zkutil::ZooKeeperPtr & zookeeper,
|
||||
const String & zookeeper_block_id_path = "");
|
||||
const String & partition_id, const zkutil::ZooKeeperPtr & zookeeper,
|
||||
const String & zookeeper_block_id_path = "") const;
|
||||
|
||||
/** Wait until all replicas, including this, execute the specified action from the log.
|
||||
* If replicas are added at the same time, it can not wait the added replica .
|
||||
@ -533,10 +538,11 @@ private:
|
||||
bool getFakePartCoveringAllPartsInPartition(const String & partition_id, MergeTreePartInfo & part_info, bool for_replace_partition = false);
|
||||
|
||||
/// Check for a node in ZK. If it is, remember this information, and then immediately answer true.
|
||||
std::unordered_set<std::string> existing_nodes_cache;
|
||||
std::mutex existing_nodes_cache_mutex;
|
||||
bool existsNodeCached(const std::string & path);
|
||||
mutable std::unordered_set<std::string> existing_nodes_cache;
|
||||
mutable std::mutex existing_nodes_cache_mutex;
|
||||
bool existsNodeCached(const std::string & path) const;
|
||||
|
||||
void getClearBlocksInPartitionOps(Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num);
|
||||
/// Remove block IDs from `blocks/` in ZooKeeper for the given partition ID in the given block number range.
|
||||
void clearBlocksInPartition(
|
||||
zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num);
|
||||
@ -544,11 +550,12 @@ private:
|
||||
/// Info about how other replicas can access this one.
|
||||
ReplicatedMergeTreeAddress getReplicatedMergeTreeAddress() const;
|
||||
|
||||
bool dropPartsInPartition(zkutil::ZooKeeper & zookeeper, String & partition_id,
|
||||
StorageReplicatedMergeTree::LogEntry & entry, bool detach);
|
||||
bool dropPart(zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach);
|
||||
bool dropAllPartsInPartition(
|
||||
zkutil::ZooKeeper & zookeeper, String & partition_id, LogEntry & entry, bool detach);
|
||||
|
||||
// Partition helpers
|
||||
void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & query_context);
|
||||
void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, const Context & query_context);
|
||||
PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, const Context & query_context);
|
||||
void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & query_context);
|
||||
void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & query_context);
|
||||
@ -565,6 +572,11 @@ private:
|
||||
MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const override;
|
||||
|
||||
void startBackgroundMovesIfNeeded() override;
|
||||
|
||||
std::set<String> getPartitionIdsAffectedByCommands(const MutationCommands & commands, const Context & query_context) const;
|
||||
PartitionBlockNumbersHolder allocateBlockNumbersInAffectedPartitions(
|
||||
const MutationCommands & commands, const Context & query_context, const zkutil::ZooKeeperPtr & zookeeper) const;
|
||||
|
||||
protected:
|
||||
/** If not 'attach', either creates a new table in ZK, or adds a replica to an existing table.
|
||||
*/
|
||||
|
@ -3,6 +3,7 @@
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/evaluateConstantExpression.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
|
||||
#include <IO/ReadHelpers.h>
|
||||
@ -32,6 +33,7 @@ IStorageURLBase::IStorageURLBase(
|
||||
const Context & context_,
|
||||
const StorageID & table_id_,
|
||||
const String & format_name_,
|
||||
const std::optional<FormatSettings> & format_settings_,
|
||||
const ColumnsDescription & columns_,
|
||||
const ConstraintsDescription & constraints_,
|
||||
const String & compression_method_)
|
||||
@ -40,6 +42,7 @@ IStorageURLBase::IStorageURLBase(
|
||||
, context_global(context_)
|
||||
, compression_method(compression_method_)
|
||||
, format_name(format_name_)
|
||||
, format_settings(format_settings_)
|
||||
{
|
||||
context_global.getRemoteHostFilter().checkURL(uri);
|
||||
|
||||
@ -58,6 +61,7 @@ namespace
|
||||
const std::string & method,
|
||||
std::function<void(std::ostream &)> callback,
|
||||
const String & format,
|
||||
const std::optional<FormatSettings> & format_settings,
|
||||
String name_,
|
||||
const Block & sample_block,
|
||||
const Context & context,
|
||||
@ -96,8 +100,10 @@ namespace
|
||||
context.getRemoteHostFilter()),
|
||||
compression_method);
|
||||
|
||||
reader = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size);
|
||||
reader = std::make_shared<AddingDefaultsBlockInputStream>(reader, columns, context);
|
||||
reader = FormatFactory::instance().getInput(format, *read_buf,
|
||||
sample_block, context, max_block_size, format_settings);
|
||||
reader = std::make_shared<AddingDefaultsBlockInputStream>(reader,
|
||||
columns, context);
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
@ -134,6 +140,7 @@ namespace
|
||||
|
||||
StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri,
|
||||
const String & format,
|
||||
const std::optional<FormatSettings> & format_settings,
|
||||
const Block & sample_block_,
|
||||
const Context & context,
|
||||
const ConnectionTimeouts & timeouts,
|
||||
@ -143,7 +150,8 @@ StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri,
|
||||
write_buf = wrapWriteBufferWithCompressionMethod(
|
||||
std::make_unique<WriteBufferFromHTTP>(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts),
|
||||
compression_method, 3);
|
||||
writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context);
|
||||
writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block,
|
||||
context, {} /* write callback */, format_settings);
|
||||
}
|
||||
|
||||
|
||||
@ -214,6 +222,7 @@ Pipe IStorageURLBase::read(
|
||||
column_names, metadata_snapshot, query_info,
|
||||
context, processed_stage, max_block_size),
|
||||
format_name,
|
||||
format_settings,
|
||||
getName(),
|
||||
getHeaderBlock(column_names, metadata_snapshot),
|
||||
context,
|
||||
@ -225,8 +234,8 @@ Pipe IStorageURLBase::read(
|
||||
|
||||
BlockOutputStreamPtr IStorageURLBase::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/)
|
||||
{
|
||||
return std::make_shared<StorageURLBlockOutputStream>(
|
||||
uri, format_name, metadata_snapshot->getSampleBlock(), context_global,
|
||||
return std::make_shared<StorageURLBlockOutputStream>(uri, format_name,
|
||||
format_settings, metadata_snapshot->getSampleBlock(), context_global,
|
||||
ConnectionTimeouts::getHTTPTimeouts(context_global),
|
||||
chooseCompressionMethod(uri.toString(), compression_method));
|
||||
}
|
||||
@ -255,16 +264,52 @@ void registerStorageURL(StorageFactory & factory)
|
||||
{
|
||||
engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], args.local_context);
|
||||
compression_method = engine_args[2]->as<ASTLiteral &>().value.safeGet<String>();
|
||||
} else compression_method = "auto";
|
||||
}
|
||||
else
|
||||
{
|
||||
compression_method = "auto";
|
||||
}
|
||||
|
||||
// Use format settings from global server context + settings from
|
||||
// the SETTINGS clause of the create query. Settings from current
|
||||
// session and user are ignored.
|
||||
FormatSettings format_settings;
|
||||
if (args.storage_def->settings)
|
||||
{
|
||||
FormatFactorySettings user_format_settings;
|
||||
|
||||
// Apply changed settings from global context, but ignore the
|
||||
// unknown ones, because we only have the format settings here.
|
||||
const auto & changes = args.context.getSettingsRef().changes();
|
||||
for (const auto & change : changes)
|
||||
{
|
||||
if (user_format_settings.has(change.name))
|
||||
{
|
||||
user_format_settings.set(change.name, change.value);
|
||||
}
|
||||
}
|
||||
|
||||
// Apply changes from SETTINGS clause, with validation.
|
||||
user_format_settings.applyChanges(args.storage_def->settings->changes);
|
||||
|
||||
format_settings = getFormatSettings(args.context,
|
||||
user_format_settings);
|
||||
}
|
||||
else
|
||||
{
|
||||
format_settings = getFormatSettings(args.context);
|
||||
}
|
||||
|
||||
return StorageURL::create(
|
||||
uri,
|
||||
args.table_id,
|
||||
format_name,
|
||||
format_settings,
|
||||
args.columns, args.constraints, args.context,
|
||||
compression_method);
|
||||
},
|
||||
{
|
||||
.supports_settings = true,
|
||||
.source_access_type = AccessType::URL,
|
||||
});
|
||||
}
|
||||
|
@ -36,6 +36,7 @@ protected:
|
||||
const Context & context_,
|
||||
const StorageID & id_,
|
||||
const String & format_name_,
|
||||
const std::optional<FormatSettings> & format_settings_,
|
||||
const ColumnsDescription & columns_,
|
||||
const ConstraintsDescription & constraints_,
|
||||
const String & compression_method_);
|
||||
@ -44,6 +45,11 @@ protected:
|
||||
const Context & context_global;
|
||||
String compression_method;
|
||||
String format_name;
|
||||
// For URL engine, we use format settings from server context + `SETTINGS`
|
||||
// clause of the `CREATE` query. In this case, format_settings is set.
|
||||
// For `url` table function, we use settings from current query context.
|
||||
// In this case, format_settings is not set.
|
||||
std::optional<FormatSettings> format_settings;
|
||||
|
||||
private:
|
||||
virtual std::string getReadMethod() const;
|
||||
@ -73,6 +79,7 @@ public:
|
||||
StorageURLBlockOutputStream(
|
||||
const Poco::URI & uri,
|
||||
const String & format,
|
||||
const std::optional<FormatSettings> & format_settings,
|
||||
const Block & sample_block_,
|
||||
const Context & context,
|
||||
const ConnectionTimeouts & timeouts,
|
||||
@ -97,15 +104,16 @@ class StorageURL final : public ext::shared_ptr_helper<StorageURL>, public IStor
|
||||
{
|
||||
friend struct ext::shared_ptr_helper<StorageURL>;
|
||||
public:
|
||||
StorageURL(
|
||||
const Poco::URI & uri_,
|
||||
const StorageID & table_id_,
|
||||
const String & format_name_,
|
||||
const ColumnsDescription & columns_,
|
||||
const ConstraintsDescription & constraints_,
|
||||
Context & context_,
|
||||
const String & compression_method_)
|
||||
: IStorageURLBase(uri_, context_, table_id_, format_name_, columns_, constraints_, compression_method_)
|
||||
StorageURL(const Poco::URI & uri_,
|
||||
const StorageID & table_id_,
|
||||
const String & format_name_,
|
||||
const std::optional<FormatSettings> & format_settings_,
|
||||
const ColumnsDescription & columns_,
|
||||
const ConstraintsDescription & constraints_,
|
||||
Context & context_,
|
||||
const String & compression_method_)
|
||||
: IStorageURLBase(uri_, context_, table_id_, format_name_,
|
||||
format_settings_, columns_, constraints_, compression_method_)
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -1,17 +1,18 @@
|
||||
#include "StorageXDBC.h"
|
||||
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/evaluateConstantExpression.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
#include <Poco/Path.h>
|
||||
#include <Processors/Pipe.h>
|
||||
#include <Storages/StorageFactory.h>
|
||||
#include <Storages/StorageURL.h>
|
||||
#include <Storages/transformQueryForExternalDatabase.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
#include <Poco/Path.h>
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
|
||||
#include <Processors/Pipe.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -33,6 +34,7 @@ StorageXDBC::StorageXDBC(
|
||||
context_,
|
||||
table_id_,
|
||||
IXDBCBridgeHelper::DEFAULT_FORMAT,
|
||||
getFormatSettings(context_),
|
||||
columns_,
|
||||
ConstraintsDescription{},
|
||||
"" /* CompressionMethod */)
|
||||
@ -121,6 +123,7 @@ BlockOutputStreamPtr StorageXDBC::write(const ASTPtr & /*query*/, const StorageM
|
||||
return std::make_shared<StorageURLBlockOutputStream>(
|
||||
request_uri,
|
||||
format_name,
|
||||
getFormatSettings(context),
|
||||
metadata_snapshot->getSampleBlock(),
|
||||
context,
|
||||
ConnectionTimeouts::getHTTPTimeouts(context),
|
||||
|
@ -1,18 +1,29 @@
|
||||
#include <Storages/StorageFile.h>
|
||||
#include <Storages/ColumnsDescription.h>
|
||||
#include <Access/AccessFlags.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
#include <TableFunctions/TableFunctionFile.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
#include "registerTableFunctions.h"
|
||||
#include <Access/AccessFlags.h>
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Storages/ColumnsDescription.h>
|
||||
#include <Storages/StorageFile.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
StoragePtr TableFunctionFile::getStorage(
|
||||
const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context,
|
||||
const std::string & table_name, const std::string & compression_method_) const
|
||||
StoragePtr TableFunctionFile::getStorage(const String & source,
|
||||
const String & format_, const ColumnsDescription & columns,
|
||||
Context & global_context, const std::string & table_name,
|
||||
const std::string & compression_method_) const
|
||||
{
|
||||
StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), format_, compression_method_, columns, ConstraintsDescription{}, global_context};
|
||||
// For `file` table function, we are going to use format settings from the
|
||||
// query context.
|
||||
StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name),
|
||||
format_,
|
||||
std::nullopt /*format settings*/,
|
||||
compression_method_,
|
||||
columns,
|
||||
ConstraintsDescription{},
|
||||
global_context};
|
||||
|
||||
return StorageFile::create(source, global_context.getUserFilesPath(), args);
|
||||
}
|
||||
|
@ -1,10 +1,12 @@
|
||||
#include <Storages/StorageURL.h>
|
||||
#include <Storages/ColumnsDescription.h>
|
||||
#include <Access/AccessFlags.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
#include <TableFunctions/TableFunctionURL.h>
|
||||
#include <Poco/URI.h>
|
||||
|
||||
#include "registerTableFunctions.h"
|
||||
#include <Access/AccessFlags.h>
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <Poco/URI.h>
|
||||
#include <Storages/ColumnsDescription.h>
|
||||
#include <Storages/StorageURL.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -14,8 +16,9 @@ StoragePtr TableFunctionURL::getStorage(
|
||||
const std::string & table_name, const String & compression_method_) const
|
||||
{
|
||||
Poco::URI uri(source);
|
||||
return StorageURL::create(uri, StorageID(getDatabaseName(), table_name), format_, columns, ConstraintsDescription{},
|
||||
global_context, compression_method_);
|
||||
return StorageURL::create(uri, StorageID(getDatabaseName(), table_name),
|
||||
format_, std::nullopt /*format settings*/, columns,
|
||||
ConstraintsDescription{}, global_context, compression_method_);
|
||||
}
|
||||
|
||||
void registerTableFunctionURL(TableFunctionFactory & factory)
|
||||
|
@ -0,0 +1,16 @@
|
||||
<yandex>
|
||||
<remote_servers>
|
||||
<test_cluster>
|
||||
<shard>
|
||||
<replica>
|
||||
<host>node1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>node2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</test_cluster>
|
||||
</remote_servers>
|
||||
</yandex>
|
@ -0,0 +1,17 @@
|
||||
<yandex>
|
||||
<shutdown_wait_unfinished>3</shutdown_wait_unfinished>
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<log>/var/log/clickhouse-server/log.log</log>
|
||||
<errorlog>/var/log/clickhouse-server/log.err.log</errorlog>
|
||||
<size>1000M</size>
|
||||
<count>10</count>
|
||||
<stderr>/var/log/clickhouse-server/stderr.log</stderr>
|
||||
<stdout>/var/log/clickhouse-server/stdout.log</stdout>
|
||||
</logger>
|
||||
<part_log>
|
||||
<database>system</database>
|
||||
<table>part_log</table>
|
||||
<flush_interval_milliseconds>500</flush_interval_milliseconds>
|
||||
</part_log>
|
||||
</yandex>
|
@ -0,0 +1,98 @@
|
||||
import pytest
|
||||
import helpers.client
|
||||
import helpers.cluster
|
||||
|
||||
|
||||
cluster = helpers.cluster.ClickHouseCluster(__file__)
|
||||
|
||||
node1 = cluster.add_instance('node1', main_configs=['configs/logs_config.xml', 'configs/cluster.xml'],
|
||||
with_zookeeper=True, stay_alive=True)
|
||||
|
||||
node2 = cluster.add_instance('node2', main_configs=['configs/logs_config.xml', 'configs/cluster.xml'],
|
||||
with_zookeeper=True, stay_alive=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def test_trivial_alter_in_partition_merge_tree_without_where(started_cluster):
|
||||
try:
|
||||
name = "test_trivial_alter_in_partition_merge_tree_without_where"
|
||||
node1.query("DROP TABLE IF EXISTS {}".format(name))
|
||||
node1.query("CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format(name))
|
||||
node1.query("INSERT INTO {} VALUES (1, 2), (2, 3)".format(name))
|
||||
with pytest.raises(helpers.client.QueryRuntimeException):
|
||||
node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 SETTINGS mutations_sync = 2".format(name))
|
||||
assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"]
|
||||
with pytest.raises(helpers.client.QueryRuntimeException):
|
||||
node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 SETTINGS mutations_sync = 2".format(name))
|
||||
assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"]
|
||||
with pytest.raises(helpers.client.QueryRuntimeException):
|
||||
node1.query("ALTER TABLE {} DELETE IN PARTITION 1 SETTINGS mutations_sync = 2".format(name))
|
||||
assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"]
|
||||
with pytest.raises(helpers.client.QueryRuntimeException):
|
||||
node1.query("ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format(name))
|
||||
assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"]
|
||||
finally:
|
||||
node1.query("DROP TABLE IF EXISTS {}".format(name))
|
||||
|
||||
|
||||
def test_trivial_alter_in_partition_merge_tree_with_where(started_cluster):
|
||||
try:
|
||||
name = "test_trivial_alter_in_partition_merge_tree_with_where"
|
||||
node1.query("DROP TABLE IF EXISTS {}".format(name))
|
||||
node1.query("CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format(name))
|
||||
node1.query("INSERT INTO {} VALUES (1, 2), (2, 3)".format(name))
|
||||
node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name))
|
||||
assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"]
|
||||
node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name))
|
||||
assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"]
|
||||
node1.query("ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name))
|
||||
assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"]
|
||||
node1.query("ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name))
|
||||
assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"]
|
||||
finally:
|
||||
node1.query("DROP TABLE IF EXISTS {}".format(name))
|
||||
|
||||
|
||||
def test_trivial_alter_in_partition_replicated_merge_tree(started_cluster):
|
||||
try:
|
||||
name = "test_trivial_alter_in_partition_replicated_merge_tree"
|
||||
|
||||
node1.query("DROP TABLE IF EXISTS {}".format(name))
|
||||
node2.query("DROP TABLE IF EXISTS {}".format(name))
|
||||
|
||||
for node in (node1, node2):
|
||||
node.query(
|
||||
"CREATE TABLE {name} (p Int64, x Int64) ENGINE=ReplicatedMergeTree('/clickhouse/{name}', '{{instance}}') ORDER BY tuple() PARTITION BY p"
|
||||
.format(name=name))
|
||||
|
||||
node1.query("INSERT INTO {} VALUES (1, 2)".format(name))
|
||||
node2.query("INSERT INTO {} VALUES (2, 3)".format(name))
|
||||
|
||||
node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE 1 SETTINGS mutations_sync = 2".format(name))
|
||||
for node in (node1, node2):
|
||||
assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"]
|
||||
node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name))
|
||||
for node in (node1, node2):
|
||||
assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"]
|
||||
with pytest.raises(helpers.client.QueryRuntimeException):
|
||||
node1.query("ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format(name))
|
||||
for node in (node1, node2):
|
||||
assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"]
|
||||
node1.query("ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name))
|
||||
for node in (node1, node2):
|
||||
assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"]
|
||||
node1.query("ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name))
|
||||
for node in (node1, node2):
|
||||
assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"]
|
||||
finally:
|
||||
node1.query("DROP TABLE IF EXISTS {}".format(name))
|
||||
node2.query("DROP TABLE IF EXISTS {}".format(name))
|
@ -1,10 +1,10 @@
|
||||
<test>
|
||||
|
||||
|
||||
<create_query>DROP TABLE IF EXISTS test_bf</create_query>
|
||||
<create_query>CREATE TABLE test_bf (`id` int, `ary` Array(String), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 8192) ENGINE = MergeTree() ORDER BY id</create_query>
|
||||
<create_query>SYSTEM STOP MERGES</create_query>
|
||||
|
||||
<query>INSERT INTO test_bf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000)</query>
|
||||
|
||||
<drop_query>SYSTEM START MERGES</drop_query>
|
||||
<drop_query>DROP TABLE IF EXISTS test_bf</drop_query>
|
||||
</test>
|
11
tests/performance/bloom_filter_select.xml
Normal file
11
tests/performance/bloom_filter_select.xml
Normal file
@ -0,0 +1,11 @@
|
||||
<test>
|
||||
<create_query>DROP TABLE IF EXISTS test_bf</create_query>
|
||||
<create_query>CREATE TABLE test_bf_indexOf (`id` int, `ary` Array(LowCardinality(Nullable(String))), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY id</create_query>
|
||||
|
||||
<fill_query>INSERT INTO test_bf_indexOf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000)</fill_query>
|
||||
|
||||
<query short="1">SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') = 2</query>
|
||||
<query short="1">SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') > 0</query>
|
||||
|
||||
<drop_query>DROP TABLE IF EXISTS test_bf_indexOf</drop_query>
|
||||
</test>
|
@ -178,3 +178,39 @@
|
||||
5000
|
||||
5000
|
||||
5000
|
||||
2
|
||||
1
|
||||
2
|
||||
1
|
||||
1
|
||||
2
|
||||
1
|
||||
2
|
||||
1
|
||||
2
|
||||
2
|
||||
1
|
||||
1
|
||||
2
|
||||
2
|
||||
1
|
||||
2
|
||||
1
|
||||
2
|
||||
2
|
||||
1
|
||||
1
|
||||
2
|
||||
1
|
||||
2
|
||||
1
|
||||
1
|
||||
1
|
||||
2
|
||||
1
|
||||
2
|
||||
1
|
||||
1
|
||||
1 value1
|
||||
1 value2
|
||||
2 value3
|
||||
|
@ -302,3 +302,54 @@ CREATE TABLE bloom_filter_array_offsets_i (order_key int, i Array(int), INDEX id
|
||||
INSERT INTO bloom_filter_array_offsets_i SELECT number AS i, if(i%2, [99999], []) FROM system.numbers LIMIT 10000;
|
||||
SELECT count() FROM bloom_filter_array_offsets_i WHERE has(i, 99999);
|
||||
DROP TABLE IF EXISTS bloom_filter_array_offsets_i;
|
||||
|
||||
DROP TABLE IF EXISTS test_bf_indexOf;
|
||||
CREATE TABLE test_bf_indexOf ( `id` int, `ary` Array(LowCardinality(Nullable(String))), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 1;
|
||||
INSERT INTO test_bf_indexOf VALUES (1, ['value1', 'value2']);
|
||||
INSERT INTO test_bf_indexOf VALUES (2, ['value3']);
|
||||
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') = 0 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') = 1 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value2') = 0 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value2') = 2 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') = 0 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') = 1 ORDER BY id FORMAT TSV;
|
||||
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') != 0 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') != 1 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value2') != 0 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value2') != 2 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') != 0 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') != 1 ORDER BY id FORMAT TSV;
|
||||
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') = 2 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') = 2 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') = 1 OR indexOf(ary, 'value3') = 1 ORDER BY id FORMAT TSV;
|
||||
|
||||
SELECT id FROM test_bf_indexOf WHERE not(indexOf(ary, 'value1')) ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE not(indexOf(ary, 'value1') == 0) ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE not(indexOf(ary, 'value1') == 1) ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE not(indexOf(ary, 'value1') == 2) ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') in (0) ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') in (1) ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') in (2) ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') not in (0) ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') not in (1) ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') not in (2) ORDER BY id FORMAT TSV;
|
||||
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') > 0 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE 0 < indexOf(ary, 'value1') ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') >= 0 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE 0 <= indexOf(ary, 'value1') ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') > 1 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE 1 < indexOf(ary, 'value1') ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') >= 1 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE 1 <= indexOf(ary, 'value1') ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') >= 2 ORDER BY id FORMAT TSV;
|
||||
SELECT id FROM test_bf_indexOf WHERE 2 <= indexOf(ary, 'value1') ORDER BY id FORMAT TSV;
|
||||
|
||||
SELECT id, ary[indexOf(ary, 'value1')] FROM test_bf_indexOf WHERE ary[indexOf(ary, 'value1')] = 'value1' ORDER BY id FORMAT TSV;
|
||||
SELECT id, ary[indexOf(ary, 'value2')] FROM test_bf_indexOf WHERE ary[indexOf(ary, 'value2')] = 'value2' ORDER BY id FORMAT TSV;
|
||||
SELECT id, ary[indexOf(ary, 'value3')] FROM test_bf_indexOf WHERE ary[indexOf(ary, 'value3')] = 'value3' ORDER BY id FORMAT TSV;
|
||||
|
||||
DROP TABLE IF EXISTS test_bf_indexOf;
|
||||
|
16
tests/queries/0_stateless/01451_detach_drop_part.reference
Normal file
16
tests/queries/0_stateless/01451_detach_drop_part.reference
Normal file
@ -0,0 +1,16 @@
|
||||
0
|
||||
1
|
||||
2
|
||||
0
|
||||
2
|
||||
all_2_2_0
|
||||
0
|
||||
1
|
||||
2
|
||||
-- drop part --
|
||||
0
|
||||
2
|
||||
-- resume merges --
|
||||
0
|
||||
2
|
||||
all_1_3_1
|
42
tests/queries/0_stateless/01451_detach_drop_part.sql
Normal file
42
tests/queries/0_stateless/01451_detach_drop_part.sql
Normal file
@ -0,0 +1,42 @@
|
||||
DROP TABLE IF EXISTS mt;
|
||||
|
||||
CREATE TABLE mt (v UInt8) ENGINE = MergeTree() order by tuple();
|
||||
SYSTEM STOP MERGES;
|
||||
|
||||
INSERT INTO mt VALUES (0);
|
||||
INSERT INTO mt VALUES (1);
|
||||
INSERT INTO mt VALUES (2);
|
||||
|
||||
SELECT v FROM mt ORDER BY v;
|
||||
|
||||
ALTER TABLE mt DETACH PART 'all_100_100_0'; -- { serverError 232 }
|
||||
|
||||
ALTER TABLE mt DETACH PART 'all_2_2_0';
|
||||
|
||||
SELECT v FROM mt ORDER BY v;
|
||||
|
||||
SELECT name FROM system.detached_parts WHERE table = 'mt';
|
||||
|
||||
ALTER TABLE mt ATTACH PART 'all_2_2_0';
|
||||
|
||||
SELECT v FROM mt ORDER BY v;
|
||||
|
||||
SELECT name FROM system.detached_parts WHERE table = 'mt';
|
||||
|
||||
SELECT '-- drop part --';
|
||||
|
||||
ALTER TABLE mt DROP PART 'all_4_4_0';
|
||||
|
||||
ALTER TABLE mt ATTACH PART 'all_4_4_0'; -- { serverError 233 }
|
||||
|
||||
SELECT v FROM mt ORDER BY v;
|
||||
|
||||
SELECT '-- resume merges --';
|
||||
SYSTEM START MERGES;
|
||||
OPTIMIZE TABLE mt FINAL;
|
||||
|
||||
SELECT v FROM mt ORDER BY v;
|
||||
|
||||
SELECT name FROM system.parts WHERE table = 'mt' AND active;
|
||||
|
||||
DROP TABLE mt;
|
@ -0,0 +1,5 @@
|
||||
all_0_0_0
|
||||
all_2_2_0
|
||||
1
|
||||
all_2_2_0
|
||||
1
|
@ -0,0 +1,49 @@
|
||||
SET replication_alter_partitions_sync = 2;
|
||||
|
||||
|
||||
DROP TABLE IF EXISTS replica1;
|
||||
DROP TABLE IF EXISTS replica2;
|
||||
|
||||
CREATE TABLE replica1 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/quorum', 'r1') order by tuple() settings max_replicated_merges_in_queue = 0;
|
||||
CREATE TABLE replica2 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/quorum', 'r2') order by tuple() settings max_replicated_merges_in_queue = 0;
|
||||
|
||||
INSERT INTO replica1 VALUES (0);
|
||||
|
||||
SYSTEM SYNC REPLICA replica2;
|
||||
|
||||
SELECT name FROM system.parts WHERE table = 'replica2' and database = currentDatabase() and active = 1;
|
||||
|
||||
ALTER TABLE replica2 DETACH PART 'all_0_0_0';
|
||||
|
||||
SELECT * FROM replica1;
|
||||
|
||||
SELECT * FROM replica2;
|
||||
|
||||
-- drop of empty partition works
|
||||
ALTER TABLE replica2 DROP PARTITION ID 'all';
|
||||
|
||||
SET insert_quorum=2;
|
||||
|
||||
INSERT INTO replica2 VALUES (1);
|
||||
|
||||
SYSTEM SYNC REPLICA replica2;
|
||||
|
||||
ALTER TABLE replica1 DETACH PART 'all_2_2_0'; --{serverError 48}
|
||||
|
||||
SELECT name FROM system.parts WHERE table = 'replica1' and database = currentDatabase() and active = 1 ORDER BY name;
|
||||
|
||||
SELECT COUNT() FROM replica1;
|
||||
|
||||
SET insert_quorum_parallel=1;
|
||||
|
||||
INSERT INTO replica2 VALUES (2);
|
||||
|
||||
-- should work, parallel quorum nodes exists only during insert
|
||||
ALTER TABLE replica1 DROP PART 'all_3_3_0';
|
||||
|
||||
SELECT name FROM system.parts WHERE table = 'replica1' and database = currentDatabase() and active = 1 ORDER BY name;
|
||||
|
||||
SELECT COUNT() FROM replica1;
|
||||
|
||||
DROP TABLE IF EXISTS replica1;
|
||||
DROP TABLE IF EXISTS replica2;
|
@ -0,0 +1,16 @@
|
||||
0
|
||||
1
|
||||
2
|
||||
0
|
||||
2
|
||||
all_1_1_0
|
||||
0
|
||||
1
|
||||
2
|
||||
-- drop part --
|
||||
0
|
||||
2
|
||||
-- resume merges --
|
||||
0
|
||||
2
|
||||
all_0_2_1
|
@ -0,0 +1,49 @@
|
||||
SET replication_alter_partitions_sync = 2;
|
||||
|
||||
DROP TABLE IF EXISTS replica1;
|
||||
DROP TABLE IF EXISTS replica2;
|
||||
|
||||
CREATE TABLE replica1 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/attach', 'r1') order by tuple() settings max_replicated_merges_in_queue = 0;
|
||||
CREATE TABLE replica2 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/attach', 'r2') order by tuple() settings max_replicated_merges_in_queue = 0;
|
||||
|
||||
INSERT INTO replica1 VALUES (0);
|
||||
INSERT INTO replica1 VALUES (1);
|
||||
INSERT INTO replica1 VALUES (2);
|
||||
|
||||
ALTER TABLE replica1 DETACH PART 'all_100_100_0'; -- { serverError 232 }
|
||||
|
||||
SELECT v FROM replica1 ORDER BY v;
|
||||
|
||||
SYSTEM SYNC REPLICA replica2;
|
||||
ALTER TABLE replica2 DETACH PART 'all_1_1_0';
|
||||
|
||||
SELECT v FROM replica1 ORDER BY v;
|
||||
|
||||
SELECT name FROM system.detached_parts WHERE table = 'replica2';
|
||||
|
||||
ALTER TABLE replica2 ATTACH PART 'all_1_1_0';
|
||||
|
||||
SYSTEM SYNC REPLICA replica1;
|
||||
SELECT v FROM replica1 ORDER BY v;
|
||||
|
||||
SELECT name FROM system.detached_parts WHERE table = 'replica2';
|
||||
|
||||
SELECT '-- drop part --';
|
||||
|
||||
ALTER TABLE replica1 DROP PART 'all_3_3_0';
|
||||
|
||||
ALTER TABLE replica1 ATTACH PART 'all_3_3_0'; -- { serverError 233 }
|
||||
|
||||
SELECT v FROM replica1 ORDER BY v;
|
||||
|
||||
SELECT '-- resume merges --';
|
||||
|
||||
ALTER TABLE replica1 MODIFY SETTING max_replicated_merges_in_queue = 1;
|
||||
OPTIMIZE TABLE replica1 FINAL;
|
||||
|
||||
SELECT v FROM replica1 ORDER BY v;
|
||||
|
||||
SELECT name FROM system.parts WHERE table = 'replica2' AND active;
|
||||
|
||||
DROP TABLE replica1;
|
||||
DROP TABLE replica2;
|
@ -0,0 +1,80 @@
|
||||
DateTime
|
||||
2020-10-15 12:00:00
|
||||
2020-10-16 00:00:00
|
||||
|
||||
2020-10-15 00:00:00
|
||||
|
||||
2020-10-15 12:00:00
|
||||
2020-10-16 00:00:00
|
||||
|
||||
2020-10-15 00:00:00
|
||||
2020-10-15 12:00:00
|
||||
2020-10-16 00:00:00
|
||||
|
||||
2020-10-15 00:00:00
|
||||
2020-10-15 12:00:00
|
||||
|
||||
2020-10-15 00:00:00
|
||||
2020-10-15 12:00:00
|
||||
2020-10-16 00:00:00
|
||||
|
||||
|
||||
2020-10-15 12:00:00
|
||||
2020-10-16 00:00:00
|
||||
|
||||
2020-10-15 00:00:00
|
||||
|
||||
2020-10-15 12:00:00
|
||||
2020-10-16 00:00:00
|
||||
|
||||
2020-10-15 00:00:00
|
||||
2020-10-15 12:00:00
|
||||
2020-10-16 00:00:00
|
||||
|
||||
2020-10-15 00:00:00
|
||||
2020-10-15 12:00:00
|
||||
|
||||
2020-10-15 00:00:00
|
||||
2020-10-15 12:00:00
|
||||
2020-10-16 00:00:00
|
||||
|
||||
DateTime64
|
||||
2020-10-15 12:00:00.000
|
||||
2020-10-16 00:00:00.000
|
||||
|
||||
2020-10-15 00:00:00.000
|
||||
|
||||
2020-10-15 12:00:00.000
|
||||
2020-10-16 00:00:00.000
|
||||
|
||||
2020-10-15 00:00:00.000
|
||||
2020-10-15 12:00:00.000
|
||||
2020-10-16 00:00:00.000
|
||||
|
||||
2020-10-15 00:00:00.000
|
||||
2020-10-15 12:00:00.000
|
||||
|
||||
2020-10-15 00:00:00.000
|
||||
2020-10-15 12:00:00.000
|
||||
2020-10-16 00:00:00.000
|
||||
|
||||
|
||||
2020-10-15 12:00:00.000
|
||||
2020-10-16 00:00:00.000
|
||||
|
||||
2020-10-15 00:00:00.000
|
||||
|
||||
2020-10-15 12:00:00.000
|
||||
2020-10-16 00:00:00.000
|
||||
|
||||
2020-10-15 00:00:00.000
|
||||
2020-10-15 12:00:00.000
|
||||
2020-10-16 00:00:00.000
|
||||
|
||||
2020-10-15 00:00:00.000
|
||||
2020-10-15 12:00:00.000
|
||||
|
||||
2020-10-15 00:00:00.000
|
||||
2020-10-15 12:00:00.000
|
||||
2020-10-16 00:00:00.000
|
||||
|
@ -0,0 +1,70 @@
|
||||
DROP TABLE IF EXISTS test;
|
||||
|
||||
CREATE TABLE test(timestamp DateTime) ENGINE = MergeTree ORDER BY timestamp;
|
||||
|
||||
INSERT INTO test VALUES ('2020-10-15 00:00:00');
|
||||
INSERT INTO test VALUES ('2020-10-15 12:00:00');
|
||||
INSERT INTO test VALUES ('2020-10-16 00:00:00');
|
||||
|
||||
SELECT 'DateTime';
|
||||
SELECT * FROM test WHERE timestamp != '2020-10-15' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp == '2020-10-15' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp > '2020-10-15' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp >= '2020-10-15' ORDER by timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp < '2020-10-16' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp <= '2020-10-16' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-15' != timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-15' == timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-15' < timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-15' <= timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-16' > timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-16' >= timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
|
||||
DROP TABLE test;
|
||||
CREATE TABLE test(timestamp DateTime64) ENGINE = MergeTree ORDER BY timestamp;
|
||||
|
||||
INSERT INTO test VALUES ('2020-10-15 00:00:00');
|
||||
INSERT INTO test VALUES ('2020-10-15 12:00:00');
|
||||
INSERT INTO test VALUES ('2020-10-16 00:00:00');
|
||||
|
||||
SELECT 'DateTime64';
|
||||
SELECT * FROM test WHERE timestamp != '2020-10-15' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp == '2020-10-15' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp > '2020-10-15' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp >= '2020-10-15' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp < '2020-10-16' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE timestamp <= '2020-10-16' ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-15' != timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-15' == timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-15' < timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-15' <= timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-16' > timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
SELECT * FROM test WHERE '2020-10-16' >= timestamp ORDER BY timestamp;
|
||||
SELECT '';
|
||||
|
||||
DROP TABLE test;
|
@ -0,0 +1,3 @@
|
||||
1|1
|
||||
1 1
|
||||
1 2
|
27
tests/queries/0_stateless/01544_file_engine_settings.sh
Executable file
27
tests/queries/0_stateless/01544_file_engine_settings.sh
Executable file
@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
the_file="$CLICKHOUSE_TMP/01544-t.csv"
|
||||
rm -f -- "$the_file"
|
||||
|
||||
# We are going to check that format settings work for File engine,
|
||||
# by creating a table with a non-default delimiter, and reading from it.
|
||||
${CLICKHOUSE_LOCAL} --query "
|
||||
create table t(a int, b int) engine File(CSV, '$the_file') settings format_csv_delimiter = '|';
|
||||
insert into t select 1 a, 1 b;
|
||||
"
|
||||
|
||||
# See what's in the file
|
||||
cat "$the_file"
|
||||
|
||||
${CLICKHOUSE_LOCAL} --query "
|
||||
create table t(a int, b int) engine File(CSV, '$the_file') settings format_csv_delimiter = '|';
|
||||
select * from t;
|
||||
"
|
||||
|
||||
# Also check that the File engine emplicitly created by clickhouse-local
|
||||
# uses the modified settings.
|
||||
${CLICKHOUSE_LOCAL} --structure="a int, b int" --input-format=CSV --format_csv_delimiter="|" --query="select * from table" <<<"1|2"
|
@ -0,0 +1,4 @@
|
||||
1 2
|
||||
1 2
|
||||
1 2
|
||||
1 2
|
19
tests/queries/0_stateless/01545_url_file_format_settings.sql
Normal file
19
tests/queries/0_stateless/01545_url_file_format_settings.sql
Normal file
@ -0,0 +1,19 @@
|
||||
create table file_delim(a int, b int) engine File(CSV, '01545_url_file_format_settings.csv') settings format_csv_delimiter = '|';
|
||||
|
||||
truncate table file_delim;
|
||||
|
||||
insert into file_delim select 1, 2;
|
||||
|
||||
-- select 1, 2 format CSV settings format_csv_delimiter='/';
|
||||
create table url_delim(a int, b int) engine URL('http://127.0.0.1:8123/?query=select%201%2C%202%20format%20CSV%20settings%20format_csv_delimiter%3D%27/%27%3B%0A', CSV) settings format_csv_delimiter = '/';
|
||||
|
||||
select * from file_delim;
|
||||
|
||||
select * from url_delim;
|
||||
|
||||
select * from file('01545_url_file_format_settings.csv', CSV, 'a int, b int') settings format_csv_delimiter = '|';
|
||||
|
||||
select * from url('http://127.0.0.1:8123/?query=select%201%2C%202%20format%20CSV%20settings%20format_csv_delimiter%3D%27/%27%3B%0A', CSV, 'a int, b int') settings format_csv_delimiter = '/';
|
||||
|
||||
|
||||
|
@ -10,6 +10,7 @@ v20.8.2.3-stable 2020-09-08
|
||||
v20.7.4.11-stable 2020-10-09
|
||||
v20.7.3.7-stable 2020-09-18
|
||||
v20.7.2.30-stable 2020-08-31
|
||||
v20.6.9.1-stable 2020-11-10
|
||||
v20.6.8.5-stable 2020-10-12
|
||||
v20.6.7.4-stable 2020-09-18
|
||||
v20.6.6.7-stable 2020-09-11
|
||||
|
|
Loading…
Reference in New Issue
Block a user