diff --git a/.gitmodules b/.gitmodules index e489444d46c..a5cdbdc6d04 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,7 +17,6 @@ [submodule "contrib/zlib-ng"] path = contrib/zlib-ng url = https://github.com/ClickHouse-Extras/zlib-ng.git - branch = clickhouse [submodule "contrib/googletest"] path = contrib/googletest url = https://github.com/google/googletest.git @@ -191,9 +190,9 @@ path = contrib/croaring url = https://github.com/RoaringBitmap/CRoaring branch = v0.2.66 +[submodule "contrib/miniselect"] + path = contrib/miniselect + url = https://github.com/danlark1/miniselect [submodule "contrib/xz"] - path = contrib/xz - url = https://github.com/xz-mirror/xz -[submodule "contrib/fast-lzma2"] - path = contrib/fast-lzma2 - url = https://github.com/conor42/fast-lzma2 + path = contrib/xz + url = https://github.com/xz-mirror/xz diff --git a/CMakeLists.txt b/CMakeLists.txt index 783a9f80b66..182d9989dc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -445,6 +445,7 @@ include (cmake/find/brotli.cmake) include (cmake/find/protobuf.cmake) include (cmake/find/grpc.cmake) include (cmake/find/pdqsort.cmake) +include (cmake/find/miniselect.cmake) include (cmake/find/hdfs3.cmake) # uses protobuf include (cmake/find/poco.cmake) include (cmake/find/curl.cmake) diff --git a/cmake/find/miniselect.cmake b/cmake/find/miniselect.cmake new file mode 100644 index 00000000000..0a50c9bf4a8 --- /dev/null +++ b/cmake/find/miniselect.cmake @@ -0,0 +1,2 @@ +set(MINISELECT_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/miniselect/include) +message(STATUS "Using miniselect: ${MINISELECT_INCLUDE_DIR}") diff --git a/contrib/miniselect b/contrib/miniselect new file mode 160000 index 00000000000..be0af6bd0b6 --- /dev/null +++ b/contrib/miniselect @@ -0,0 +1 @@ +Subproject commit be0af6bd0b6eb044d1acc4f754b229972d99903a diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 298e80350e3..36aa8baf252 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -127,7 +127,7 @@ function clone_submodules ( cd "$FASTTEST_SOURCE" -SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11 contrib/croaring contrib/xz) +SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11 contrib/croaring contrib/miniselect) git submodule sync git submodule update --init --recursive "${SUBMODULES_TO_UPDATE[@]}" diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 874dca751f3..458f78fcdb4 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -17,13 +17,6 @@ def get_skip_list_cmd(path): return '' -def run_perf_test(cmd, xmls_path, output_folder): - output_path = os.path.join(output_folder, "perf_stress_run.txt") - f = open(output_path, 'w') - p = Popen("{} --skip-tags=long --recursive --input-files {}".format(cmd, xmls_path), shell=True, stdout=f, stderr=f) - return p - - def get_options(i): options = "" if 0 < i: @@ -75,8 +68,6 @@ if __name__ == "__main__": args = parser.parse_args() func_pipes = [] - perf_process = None - perf_process = run_perf_test(args.perf_test_cmd, args.perf_test_xml_path, args.output_folder) func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests, args.global_time_limit) logging.info("Will wait functests to finish") diff --git a/docs/en/development/adding_test_queries.md b/docs/en/development/adding_test_queries.md new file mode 100644 index 00000000000..4770d48ebd4 --- /dev/null +++ b/docs/en/development/adding_test_queries.md @@ -0,0 +1,141 @@ +# How to add test queries to ClickHouse CI + +ClickHouse has hundreds (or even thousands) of features. Every commit get checked by a complex set of tests containing many thousands of test cases. + +The core functionality is very well tested, but some corner-cases and different combinations of features can be uncovered with ClickHouse CI. + +Most of the bugs/regressions we see happen in that 'grey area' where test coverage is poor. + +And we are very interested in covering most of the possible scenarios and feature combinations used in real life by tests. + +## Why adding tests + +Why/when you should add a test case into ClickHouse code: +1) you use some complicated scenarios / feature combinations / you have some corner case which is probably not widely used +2) you see that certain behavior gets changed between version w/o notifications in the changelog +3) you just want to help to improve ClickHouse quality and ensure the features you use will not be broken in the future releases +4) once the test is added/accepted, you can be sure the corner case you check will never be accidentally broken. +5) you will be a part of great open-source community +6) your name will be visible in the `system.contributors` table! +7) you will make a world bit better :) + +### Steps to do + +#### Prerequisite + +I assume you run some Linux machine (you can use docker / virtual machines on other OS) and any modern browser / internet connection, and you have some basic Linux & SQL skills. + +Any highly specialized knowledge is not needed (so you don't need to know C++ or know something about how ClickHouse CI works). + + +#### Preparation + +1) [create GitHub account](https://github.com/join) (if you haven't one yet) +2) [setup git](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/set-up-git) +```bash +# for Ubuntu +sudo apt-get update +sudo apt-get install git + +git config --global user.name "John Doe" # fill with your name +git config --global user.email "email@example.com" # fill with your email + +``` +3) [fork ClickHouse project](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/fork-a-repo) - just open [https://github.com/ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse) and press fork button in the top right corner: +![fork repo](https://github-images.s3.amazonaws.com/help/bootcamp/Bootcamp-Fork.png) + +4) clone your fork to some folder on your PC, for example, `~/workspace/ClickHouse` +``` +mkdir ~/workspace && cd ~/workspace +git clone https://github.com/< your GitHub username>/ClickHouse +cd ClickHouse +git remote add upstream https://github.com/ClickHouse/ClickHouse +``` + +#### New branch for the test + +1) create a new branch from the latest clickhouse master +``` +cd ~/workspace/ClickHouse +git fetch upstream +git checkout -b name_for_a_branch_with_my_test upstream/master +``` + +#### Install & run clickhouse + +1) install `clickhouse-server` (follow [official docs](https://clickhouse.tech/docs/en/getting-started/install/)) +2) install test configurations (it will use Zookeeper mock implementation and adjust some settings) +``` +cd ~/workspace/ClickHouse/tests/config +sudo ./install.sh +``` +3) run clickhouse-server +``` +sudo systemctl restart clickhouse-server +``` + +#### Creating the test file + + +1) find the number for your test - find the file with the biggest number in `tests/queries/0_stateless/` + +```sh +$ cd ~/workspace/ClickHouse +$ ls tests/queries/0_stateless/[0-9]*.reference | tail -n 1 +tests/queries/0_stateless/01520_client_print_query_id.reference +``` +Currently, the last number for the test is `01520`, so my test will have the number `01521` + +2) create an SQL file with the next number and name of the feature you test + +```sh +touch tests/queries/0_stateless/01521_dummy_test.sql +``` + +3) edit SQL file with your favorite editor (see hint of creating tests below) +```sh +vim tests/queries/0_stateless/01521_dummy_test.sql +``` + + +4) run the test, and put the result of that into the reference file: +``` +clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee tests/queries/0_stateless/01521_dummy_test.reference +``` + +5) ensure everything is correct, if the test output is incorrect (due to some bug for example), adjust the reference file using text editor. + +#### How create good test + +- test should be + - minimal - create only tables related to tested functionality, remove unrelated columns and parts of query + - fast - should not take longer than few seconds (better subseconds) + - correct - fails then feature is not working + - deteministic + - isolated / stateless + - don't rely on some environment things + - don't rely on timing when possible +- try to cover corner cases (zeros / Nulls / empty sets / throwing exceptions) +- to test that query return errors, you can put special comment after the query: `-- { serverError 60 }` or `-- { clientError 20 }` +- don't switch databases (unless necessary) +- you can create several table replicas on the same node if needed +- you can use one of the test cluster definitions when needed (see system.clusters) +- use `number` / `numbers_mt` / `zeros` / `zeros_mt` and similar for queries / to initialize data when appliable +- clean up the created objects after test and before the test (DROP IF EXISTS) - in case of some dirty state +- prefer sync mode of operations (mutations, merges, etc.) +- use other SQL files in the `0_stateless` folder as an example +- ensure the feature / feature combination you want to tests is not covered yet with existsing tests + +#### Commit / push / create PR. + +1) commit & push your changes +```sh +cd ~/workspace/ClickHouse +git add tests/queries/0_stateless/01521_dummy_test.sql +git add tests/queries/0_stateless/01521_dummy_test.reference +git commit # use some nice commit message when possible +git push origin HEAD +``` +2) use a link which was shown during the push, to create a PR into the main repo +3) adjust the PR title and contents, in `Changelog category (leave one)` keep +`Build/Testing/Packaging Improvement`, fill the rest of the fields if you want. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index d83f7d6c219..f9c3c8a5d75 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -384,7 +384,7 @@ Possible values: - `'basic'` — Use basic parser. - ClickHouse can parse only the basic `YYYY-MM-DD HH:MM:SS` format. For example, `'2019-08-20 10:18:56'`. + ClickHouse can parse only the basic `YYYY-MM-DD HH:MM:SS` or `YYYY-MM-DD` format. For example, `'2019-08-20 10:18:56'` or `2019-08-20`. Default value: `'basic'`. diff --git a/docs/en/sql-reference/data-types/date.md b/docs/en/sql-reference/data-types/date.md index c686a987b2f..886e93f433c 100644 --- a/docs/en/sql-reference/data-types/date.md +++ b/docs/en/sql-reference/data-types/date.md @@ -3,10 +3,45 @@ toc_priority: 47 toc_title: Date --- -# Date {#date} +# Date {#data_type-date} A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2106, but the final fully-supported year is 2105). The date value is stored without the time zone. +## Examples {#examples} + +**1.** Creating a table with a `DateTime`-type column and inserting data into it: + +``` sql +CREATE TABLE dt +( + `timestamp` Date, + `event_id` UInt8 +) +ENGINE = TinyLog; +``` + +``` sql +INSERT INTO dt Values (1546300800, 1), ('2019-01-01', 2); +``` + +``` sql +SELECT * FROM dt; +``` + +``` text +┌──timestamp─┬─event_id─┐ +│ 2019-01-01 │ 1 │ +│ 2019-01-01 │ 2 │ +└────────────┴──────────┘ +``` + +## See Also {#see-also} + +- [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md) +- [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime) +- [`DateTime` data type](../../sql-reference/data-types/datetime.md) + + [Original article](https://clickhouse.tech/docs/en/data_types/date/) diff --git a/docs/en/sql-reference/table-functions/null.md b/docs/en/sql-reference/table-functions/null.md index 6edec61add7..355a45a83e1 100644 --- a/docs/en/sql-reference/table-functions/null.md +++ b/docs/en/sql-reference/table-functions/null.md @@ -5,7 +5,7 @@ toc_title: null function # null {#null-function} -Accepts an inserted data of the specified structure and immediately drops it away. The function is used for convenience writing tests and demonstrations. +Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately droped right after the query execution. The function is used for the convenience of test writing and demonstrations. **Syntax** @@ -19,7 +19,7 @@ null('structure') **Returned value** -A table with the specified structure, which is dropped right after the query execution. +A temporary `Null`-engine table with the specified structure. **Example** @@ -36,6 +36,8 @@ INSERT INTO t SELECT * FROM numbers_mt(1000000000); DROP TABLE IF EXISTS t; ``` -See also: format **Null**. +See also: + +- [Null table engine](../../engines/table-engines/special/null.md) [Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/null/) diff --git a/docs/ru/sql-reference/table-functions/null.md b/docs/ru/sql-reference/table-functions/null.md new file mode 100644 index 00000000000..8e0173733f8 --- /dev/null +++ b/docs/ru/sql-reference/table-functions/null.md @@ -0,0 +1,43 @@ +--- +toc_priority: 53 +toc_title: null функция +--- + +# null {#null-function} + +Создает временную таблицу указанной структуры с движком [Null](../../engines/table-engines/special/null.md). В соответствии со свойствами движка, данные в таблице игнорируются, а сама таблица удаляется сразу после выполнения запроса. Функция используется для удобства написания тестов и демонстрационных примеров. + +**Синтаксис** + +``` sql +null('structure') +``` + +**Параметр** + +- `structure` — список колонок и их типов. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +Временная таблица указанной структуры с движком `Null`. + +**Пример** + +Один запрос с функцией `null`: + +``` sql +INSERT INTO function null('x UInt64') SELECT * FROM numbers_mt(1000000000); +``` +заменяет три запроса: + +```sql +CREATE TABLE t (x UInt64) ENGINE = Null; +INSERT INTO t SELECT * FROM numbers_mt(1000000000); +DROP TABLE IF EXISTS t; +``` + +См. также: + +- [Движок таблиц Null](../../engines/table-engines/special/null.md) + +[Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/null/) diff --git a/src/AggregateFunctions/QuantileExact.h b/src/AggregateFunctions/QuantileExact.h index 3f5a0907126..de909c27565 100644 --- a/src/AggregateFunctions/QuantileExact.h +++ b/src/AggregateFunctions/QuantileExact.h @@ -8,6 +8,7 @@ #include #include +#include namespace DB { @@ -87,7 +88,7 @@ struct QuantileExact : QuantileExactBase> { size_t n = level < 1 ? level * array.size() : (array.size() - 1); - std::nth_element(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm. + miniselect::floyd_rivest_select(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm. return array[n]; } @@ -107,7 +108,7 @@ struct QuantileExact : QuantileExactBase> size_t n = level < 1 ? level * array.size() : (array.size() - 1); - std::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n, array.end()); result[indices[i]] = array[n]; prev_n = n; @@ -144,7 +145,7 @@ struct QuantileExactExclusive : public QuantileExact else if (n < 1) return static_cast(array[0]); - std::nth_element(array.begin(), array.begin() + n - 1, array.end()); + miniselect::floyd_rivest_select(array.begin(), array.begin() + n - 1, array.end()); auto nth_element = std::min_element(array.begin() + n, array.end()); return static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); @@ -173,7 +174,7 @@ struct QuantileExactExclusive : public QuantileExact result[indices[i]] = static_cast(array[0]); else { - std::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); + miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n - 1, array.end()); auto nth_element = std::min_element(array.begin() + n, array.end()); result[indices[i]] = static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); @@ -209,7 +210,7 @@ struct QuantileExactInclusive : public QuantileExact else if (n < 1) return static_cast(array[0]); - std::nth_element(array.begin(), array.begin() + n - 1, array.end()); + miniselect::floyd_rivest_select(array.begin(), array.begin() + n - 1, array.end()); auto nth_element = std::min_element(array.begin() + n, array.end()); return static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); @@ -236,7 +237,7 @@ struct QuantileExactInclusive : public QuantileExact result[indices[i]] = static_cast(array[0]); else { - std::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); + miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n - 1, array.end()); auto nth_element = std::min_element(array.begin() + n, array.end()); result[indices[i]] = static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); diff --git a/src/AggregateFunctions/QuantileTiming.h b/src/AggregateFunctions/QuantileTiming.h index 2ab8c866615..28bcde5c140 100644 --- a/src/AggregateFunctions/QuantileTiming.h +++ b/src/AggregateFunctions/QuantileTiming.h @@ -7,6 +7,7 @@ #include #include +#include namespace DB { @@ -179,7 +180,7 @@ namespace detail /// Sorting an array will not be considered a violation of constancy. auto & array = elems; - std::nth_element(array.begin(), array.begin() + n, array.end()); + miniselect::floyd_rivest_select(array.begin(), array.begin() + n, array.end()); quantile = array[n]; } @@ -200,7 +201,7 @@ namespace detail ? level * elems.size() : (elems.size() - 1); - std::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n, array.end()); result[level_index] = array[n]; prev_n = n; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6839b06677d..36e114bd916 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -321,6 +321,7 @@ target_include_directories(clickhouse_common_io PUBLIC ${CMAKE_CURRENT_BINARY_DI dbms_target_include_directories(PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/Core/include) dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) +dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${MINISELECT_INCLUDE_DIR}) if (ZSTD_LIBRARY) dbms_target_link_libraries(PRIVATE ${ZSTD_LIBRARY}) @@ -336,13 +337,6 @@ if (LZMA_LIBRARY) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${LZMA_INCLUDE_DIR}) endif() -set (FAST_LZMA_LIBRARY fast-lzma2) -set (FAST_LZMA_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/fast-lzma2/) -if (FAST_LZMA_LIBRARY) - target_link_libraries (clickhouse_common_io PUBLIC ${FAST_LZMA_LIBRARY}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${FAST_LZMA_INCLUDE_DIR}) -endif() - if (USE_ICU) dbms_target_link_libraries (PRIVATE ${ICU_LIBRARIES}) dbms_target_include_directories (SYSTEM PRIVATE ${ICU_INCLUDE_DIRS}) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 9b948236943..6dbe755f0ba 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -20,6 +20,7 @@ #include #include +#include namespace DB { @@ -782,7 +783,7 @@ void ColumnArray::getPermutationImpl(size_t limit, Permutation & res, Comparator auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; }; if (limit) - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less); else std::sort(res.begin(), res.end(), less); } @@ -835,7 +836,7 @@ void ColumnArray::updatePermutationImpl(size_t limit, Permutation & res, EqualRa /// Since then we are working inside the interval. - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); auto new_first = first; for (auto j = first + 1; j < limit; ++j) { diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index b9549175f6c..4285259a4f4 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -8,6 +8,7 @@ #include #include +#include #include @@ -162,10 +163,10 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum { const auto& [first, last] = equal_ranges[i]; if (reverse) - std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, + std::sort(res.begin() + first, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else - std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, + std::sort(res.begin() + first, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); auto new_first = first; @@ -193,10 +194,10 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum /// Since then we are working inside the interval. if (reverse) - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); auto new_first = first; diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index c33ab34b541..128b595803f 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -253,9 +254,9 @@ protected: sort_end = res.begin() + limit; if (reverse) - std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; }); + miniselect::floyd_rivest_partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; }); else - std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; }); + miniselect::floyd_rivest_partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; }); } }; diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 0e44b83791c..41e46a7fa98 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -10,6 +10,7 @@ #include #include +#include #include @@ -157,9 +158,9 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir if (limit) { if (reverse) - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); else - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); } else { @@ -217,9 +218,9 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu /// Since then we are working inside the interval. if (reverse) - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); auto new_first = first; for (auto j = first + 1; j < limit; ++j) diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 3f03734b738..9b87a409aaa 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -8,6 +8,7 @@ #include #include +#include namespace DB { @@ -393,7 +394,7 @@ void ColumnLowCardinality::updatePermutationImpl(size_t limit, Permutation & res /// Since then we are working inside the interval. - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); auto new_first = first; for (auto j = first + 1; j < limit; ++j) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 23798f64a9c..541863486a6 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -10,6 +10,7 @@ #include #include +#include namespace DB { @@ -313,7 +314,7 @@ void ColumnString::getPermutationImpl(size_t limit, Permutation & res, Comparato auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; }; if (limit) - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less); else std::sort(res.begin(), res.end(), less); } @@ -365,7 +366,7 @@ void ColumnString::updatePermutationImpl(size_t limit, Permutation & res, EqualR /// Since then we are working inside the interval. - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); size_t new_first = first; for (size_t j = first + 1; j < limit; ++j) diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index d6e1ca982d6..cbe5c7e11cd 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -9,7 +9,7 @@ #include #include #include - +#include namespace DB { @@ -352,7 +352,7 @@ void ColumnTuple::getPermutationImpl(size_t limit, Permutation & res, LessOperat if (limit) { - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less); } else { diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 733a1510f93..e9af38d6984 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -17,7 +17,7 @@ #include #include #include - +#include #ifdef __SSE2__ #include @@ -156,9 +156,9 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi res[i] = i; if (reverse) - std::partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint)); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint)); else - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint)); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint)); } else { @@ -254,9 +254,9 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire /// Since then, we are working inside the interval. if (reverse) - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint)); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint)); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint)); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint)); size_t new_first = first; for (size_t j = first + 1; j < limit; ++j) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index c8521d2f91b..580756361b1 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -399,6 +399,7 @@ class IColumn; \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ + M(UInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ M(UInt64, multiple_joins_rewriter_version, 0, "Obsolete setting, does nothing. Will be removed after 2021-03-31", 0) \ M(Bool, experimental_use_processors, true, "Obsolete setting, does nothing. Will be removed after 2020-11-29.", 0) \ M(Bool, force_optimize_skip_unused_shards_no_nested, false, "Obsolete setting, does nothing. Will be removed after 2020-12-01. Use force_optimize_skip_unused_shards_nesting instead.", 0) \ diff --git a/src/IO/LZMADeflatingWriteBuffer.cpp b/src/IO/LZMADeflatingWriteBuffer.cpp index 66cd11c13d6..378f7c4ea1b 100644 --- a/src/IO/LZMADeflatingWriteBuffer.cpp +++ b/src/IO/LZMADeflatingWriteBuffer.cpp @@ -13,10 +13,10 @@ LZMADeflatingWriteBuffer::LZMADeflatingWriteBuffer( : BufferWithOwnMemory(buf_size, existing_memory, alignment), out(std::move(out_)) { // FL2_createCStreamMt(number of threads, flag of two dictionaries usage) - lstr = FL2_createCStreamMt(2, 0); - /* size_t res = */ FL2_initCStream(lstr, compression_level); + // lstr = FL2_createCStreamMt(2, 0); + // size_t res = FL2_initCStream(lstr, compression_level); - /*lstr = LZMA_STREAM_INIT; + lstr = LZMA_STREAM_INIT; lstr.allocator = nullptr; lstr.next_in = nullptr; lstr.avail_in = 0; @@ -26,10 +26,13 @@ LZMADeflatingWriteBuffer::LZMADeflatingWriteBuffer( // options for further compression lzma_options_lzma opt_lzma2; if (lzma_lzma_preset(&opt_lzma2, compression_level)) - throw Exception( - std::string("lzma preset failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); + throw Exception(ErrorCodes::LZMA_STREAM_ENCODER_FAILED, "lzma preset failed: lzma version: {}", LZMA_VERSION_STRING); + // LZMA_FILTER_X86 - + // LZMA2 - codec for *.xz files compression; LZMA is not suitable for this purpose + // VLI - variable length integer (in *.xz most integers encoded as VLI) + // LZMA_VLI_UNKNOWN (UINT64_MAX) - VLI value to denote that the value is unknown lzma_filter filters[] = { {.id = LZMA_FILTER_X86, .options = nullptr}, {.id = LZMA_FILTER_LZMA2, .options = &opt_lzma2}, @@ -39,9 +42,10 @@ LZMADeflatingWriteBuffer::LZMADeflatingWriteBuffer( if (ret != LZMA_OK) throw Exception( - std::string("lzma stream encoder init failed: ") + std::to_string(ret) + "; lzma version: " + LZMA_VERSION_STRING, - ErrorCodes::LZMA_STREAM_ENCODER_FAILED); - */ + ErrorCodes::LZMA_STREAM_ENCODER_FAILED, + "lzma stream encoder init failed: error code: {} lzma version: {}", + ret, + LZMA_VERSION_STRING); } LZMADeflatingWriteBuffer::~LZMADeflatingWriteBuffer() @@ -50,7 +54,7 @@ LZMADeflatingWriteBuffer::~LZMADeflatingWriteBuffer() { finish(); - //lzma_end(&lstr); + lzma_end(&lstr); } catch (...) { @@ -60,7 +64,6 @@ LZMADeflatingWriteBuffer::~LZMADeflatingWriteBuffer() void LZMADeflatingWriteBuffer::nextImpl() { - /* if (!offset()) return; @@ -74,27 +77,25 @@ void LZMADeflatingWriteBuffer::nextImpl() lstr.next_out = reinterpret_cast(out->position()); lstr.avail_out = out->buffer().end() - out->position(); - lzma_ret ret = lzma_code(&lstr, action); out->position() = out->buffer().end() - lstr.avail_out; - if (ret == LZMA_STREAM_END) return; if (ret != LZMA_OK) throw Exception( - std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING, - ErrorCodes::LZMA_STREAM_ENCODER_FAILED); + ErrorCodes::LZMA_STREAM_ENCODER_FAILED, + "lzma stream encoding failed: error code: {}; lzma_version: {}", + ret, + LZMA_VERSION_STRING); } while (lstr.avail_in > 0 || lstr.avail_out == 0); - */ } void LZMADeflatingWriteBuffer::finish() { - /* if (finished) return; @@ -109,7 +110,6 @@ void LZMADeflatingWriteBuffer::finish() lzma_ret ret = lzma_code(&lstr, LZMA_FINISH); out->position() = out->buffer().end() - lstr.avail_out; - if (ret == LZMA_STREAM_END) { finished = true; @@ -118,10 +118,11 @@ void LZMADeflatingWriteBuffer::finish() if (ret != LZMA_OK) throw Exception( - std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING, - ErrorCodes::LZMA_STREAM_ENCODER_FAILED); + ErrorCodes::LZMA_STREAM_ENCODER_FAILED, + "lzma stream encoding failed: error code: {}; lzma version: {}", + ret, + LZMA_VERSION_STRING); } while (lstr.avail_out == 0); - */ } } diff --git a/src/IO/LZMADeflatingWriteBuffer.h b/src/IO/LZMADeflatingWriteBuffer.h index aadf15ec6dd..bd733bfa49b 100644 --- a/src/IO/LZMADeflatingWriteBuffer.h +++ b/src/IO/LZMADeflatingWriteBuffer.h @@ -4,7 +4,6 @@ #include #include -#include namespace DB { @@ -27,7 +26,8 @@ private: void nextImpl() override; std::unique_ptr out; - FL2_CStream * lstr; + lzma_stream lstr; + // FL2_CStream * lstr; bool finished = false; }; } diff --git a/src/IO/LZMAInflatingReadBuffer.cpp b/src/IO/LZMAInflatingReadBuffer.cpp index 09aa31b7f43..b8aa1e703cb 100644 --- a/src/IO/LZMAInflatingReadBuffer.cpp +++ b/src/IO/LZMAInflatingReadBuffer.cpp @@ -9,10 +9,6 @@ namespace ErrorCodes LZMAInflatingReadBuffer::LZMAInflatingReadBuffer(std::unique_ptr in_, size_t buf_size, char * existing_memory, size_t alignment) : BufferWithOwnMemory(buf_size, existing_memory, alignment), in(std::move(in_)), eof(false) { - // FL2_createDStreamMt(number of threads) - lstr = FL2_createDStreamMt(2); - /* size_t res = */ FL2_initDStream(lstr); - /* lstr = LZMA_STREAM_INIT; lstr.allocator = nullptr; lstr.next_in = nullptr; @@ -21,40 +17,48 @@ LZMAInflatingReadBuffer::LZMAInflatingReadBuffer(std::unique_ptr in_ lstr.avail_out = 0; // 500 mb - uint64_t memlimit = 500 << 20; + uint64_t memlimit = 500ULL << 20; lzma_ret ret = lzma_stream_decoder(&lstr, memlimit, LZMA_CONCATENATED); // lzma does not provide api for converting error code to string unlike zlib if (ret != LZMA_OK) throw Exception( - std::string("lzma_stream_decoder initialization failed: error code: ") + std::to_string(ret) - + "; lzma version: " + LZMA_VERSION_STRING, - ErrorCodes::LZMA_STREAM_DECODER_FAILED); - */ + ErrorCodes::LZMA_STREAM_DECODER_FAILED, + "lzma_stream_decoder initialization failed: error code: {}; lzma version: {}", + ret, + LZMA_VERSION_STRING); } LZMAInflatingReadBuffer::~LZMAInflatingReadBuffer() { - //lzma_end(&lstr); + lzma_end(&lstr); } bool LZMAInflatingReadBuffer::nextImpl() { - /* if (eof) return false; + lzma_action action = LZMA_RUN; + if (!lstr.avail_in) { in->nextIfAtEnd(); lstr.next_in = reinterpret_cast(in->position()); lstr.avail_in = in->buffer().end() - in->position(); } + + if (in->eof()) + { + action = LZMA_FINISH; + } + lstr.next_out = reinterpret_cast(internal_buffer.begin()); lstr.avail_out = internal_buffer.size(); + // std::cout << lstr.avail_in << " " << lstr.avail_out << std::endl; - lzma_ret ret = lzma_code(&lstr, LZMA_RUN); - + lzma_ret ret = lzma_code(&lstr, action); + // std::cout << ret << std::endl; in->position() = in->buffer().end() - lstr.avail_in; working_buffer.resize(internal_buffer.size() - lstr.avail_out); @@ -69,7 +73,7 @@ bool LZMAInflatingReadBuffer::nextImpl() { throw Exception( ErrorCodes::LZMA_STREAM_DECODER_FAILED, - "lzma decoder finished, but stream is still alive: error code: {}; lzma version: {}", + "lzma decoder finished, but input stream has not exceeded: error code: {}; lzma version: {}", ret, LZMA_VERSION_STRING); } @@ -83,7 +87,5 @@ bool LZMAInflatingReadBuffer::nextImpl() LZMA_VERSION_STRING); return true; - */ - return true; } } diff --git a/src/IO/LZMAInflatingReadBuffer.h b/src/IO/LZMAInflatingReadBuffer.h index 4a9893e5b4c..95c0c8f7681 100644 --- a/src/IO/LZMAInflatingReadBuffer.h +++ b/src/IO/LZMAInflatingReadBuffer.h @@ -4,7 +4,6 @@ #include #include -#include namespace DB { @@ -27,7 +26,8 @@ private: bool nextImpl() override; std::unique_ptr in; - FL2_DStream * lstr; + lzma_stream lstr; + // FL2_DStream * lstr; bool eof; }; diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index bf41de3959a..73ee0dfcd95 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -817,7 +817,11 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D { static constexpr bool throw_exception = std::is_same_v; + /// YYYY-MM-DD hh:mm:ss static constexpr auto date_time_broken_down_length = 19; + /// YYYY-MM-DD + static constexpr auto date_broken_down_length = 10; + /// unix timestamp max length static constexpr auto unix_timestamp_max_length = 10; char s[date_time_broken_down_length]; @@ -831,12 +835,15 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D ++buf.position(); } - /// 2015-01-01 01:02:03 + /// 2015-01-01 01:02:03 or 2015-01-01 if (s_pos == s + 4 && !buf.eof() && (*buf.position() < '0' || *buf.position() > '9')) { - const size_t remaining_size = date_time_broken_down_length - (s_pos - s); - size_t size = buf.read(s_pos, remaining_size); - if (remaining_size != size) + const auto already_read_length = s_pos - s; + const size_t remaining_date_time_size = date_time_broken_down_length - already_read_length; + const size_t remaining_date_size = date_broken_down_length - already_read_length; + + size_t size = buf.read(s_pos, remaining_date_time_size); + if (size != remaining_date_time_size && size != remaining_date_size) { s_pos[size] = 0; @@ -850,9 +857,16 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); - UInt8 hour = (s[11] - '0') * 10 + (s[12] - '0'); - UInt8 minute = (s[14] - '0') * 10 + (s[15] - '0'); - UInt8 second = (s[17] - '0') * 10 + (s[18] - '0'); + UInt8 hour = 0; + UInt8 minute = 0; + UInt8 second = 0; + + if (size == remaining_date_time_size) + { + hour = (s[11] - '0') * 10 + (s[12] - '0'); + minute = (s[14] - '0') * 10 + (s[15] - '0'); + second = (s[17] - '0') * 10 + (s[18] - '0'); + } if (unlikely(year == 0)) datetime = 0; diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 9ff1858c723..0e91bc1dd8e 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -700,7 +700,7 @@ UInt128 stringToUUID(const String & str); template ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut); -/** In YYYY-MM-DD hh:mm:ss format, according to specified time zone. +/** In YYYY-MM-DD hh:mm:ss or YYYY-MM-DD format, according to specified time zone. * As an exception, also supported parsing of unix timestamp in form of decimal number. */ template @@ -709,12 +709,17 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons /** Read 10 characters, that could represent unix timestamp. * Only unix timestamp of 5-10 characters is supported. * Then look at 5th character. If it is a number - treat whole as unix timestamp. - * If it is not a number - then parse datetime in YYYY-MM-DD hh:mm:ss format. + * If it is not a number - then parse datetime in YYYY-MM-DD hh:mm:ss or YYYY-MM-DD format. */ /// Optimistic path, when whole value is in buffer. const char * s = buf.position(); - if (s + 19 <= buf.buffer().end()) + + /// YYYY-MM-DD hh:mm:ss + static constexpr auto DateTimeStringInputSize = 19; + bool optimistic_path_for_date_time_input = s + DateTimeStringInputSize <= buf.buffer().end(); + + if (optimistic_path_for_date_time_input) { if (s[4] < '0' || s[4] > '9') { @@ -731,7 +736,7 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons else datetime = date_lut.makeDateTime(year, month, day, hour, minute, second); - buf.position() += 19; + buf.position() += DateTimeStringInputSize; return ReturnType(true); } else diff --git a/src/Interpreters/ClusterProxy/IStreamFactory.h b/src/Interpreters/ClusterProxy/IStreamFactory.h index 8829dc38c93..80be585d15e 100644 --- a/src/Interpreters/ClusterProxy/IStreamFactory.h +++ b/src/Interpreters/ClusterProxy/IStreamFactory.h @@ -16,6 +16,9 @@ struct SelectQueryInfo; class Pipe; using Pipes = std::vector; +class QueryPlan; +using QueryPlanPtr = std::unique_ptr; + namespace ClusterProxy { @@ -31,7 +34,9 @@ public: const String & query, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, const SelectQueryInfo & query_info, - Pipes & res) = 0; + std::vector & res, + Pipes & remote_pipes, + Pipes & delayed_pipes) = 0; }; } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 9e64695d1a0..bb966d57c4f 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -14,6 +14,8 @@ #include #include #include +#include + namespace ProfileEvents { @@ -69,37 +71,27 @@ SelectStreamFactory::SelectStreamFactory( namespace { -auto createLocalPipe( - const ASTPtr & query_ast, const Block & header, const Context & context, QueryProcessingStage::Enum processed_stage) +std::unique_ptr createLocalPlan( + const ASTPtr & query_ast, + const Block & header, + const Context & context, + QueryProcessingStage::Enum processed_stage) { checkStackSize(); - InterpreterSelectQuery interpreter(query_ast, context, SelectQueryOptions(processed_stage)); auto query_plan = std::make_unique(); + InterpreterSelectQuery interpreter(query_ast, context, SelectQueryOptions(processed_stage)); interpreter.buildQueryPlan(*query_plan); - auto pipeline = std::move(*query_plan->buildQueryPipeline()); - /// Avoid going it out-of-scope for EXPLAIN - pipeline.addQueryPlan(std::move(query_plan)); + /// Convert header structure to expected. + /// Also we ignore constants from result and replace it with constants from header. + /// It is needed for functions like `now64()` or `randConstant()` because their values may be different. + auto converting = std::make_unique(query_plan->getCurrentDataStream(), header, true); + converting->setStepDescription("Convert block structure for query from local replica"); + query_plan->addStep(std::move(converting)); - pipeline.addSimpleTransform([&](const Block & source_header) - { - return std::make_shared( - source_header, header, ConvertingTransform::MatchColumnsMode::Name, true); - }); - - /** Materialization is needed, since from remote servers the constants come materialized. - * If you do not do this, different types (Const and non-Const) columns will be produced in different threads, - * And this is not allowed, since all code is based on the assumption that in the block stream all types are the same. - */ - - /* Now we don't need to materialize constants, because RemoteBlockInputStream will ignore constant and take it from header. - * So, streams from different threads will always have the same header. - */ - - pipeline.setMaxThreads(1); - return QueryPipeline::getPipe(std::move(pipeline)); + return query_plan; } String formattedAST(const ASTPtr & ast) @@ -119,7 +111,9 @@ void SelectStreamFactory::createForShard( const String &, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, const SelectQueryInfo &, - Pipes & pipes) + std::vector & plans, + Pipes & remote_pipes, + Pipes & delayed_pipes) { bool add_agg_info = processed_stage == QueryProcessingStage::WithMergeableState; bool add_totals = false; @@ -136,7 +130,7 @@ void SelectStreamFactory::createForShard( auto emplace_local_stream = [&]() { - pipes.emplace_back(createLocalPipe(modified_query_ast, header, context, processed_stage)); + plans.emplace_back(createLocalPlan(modified_query_ast, header, context, processed_stage)); }; String modified_query = formattedAST(modified_query_ast); @@ -149,7 +143,7 @@ void SelectStreamFactory::createForShard( if (!table_func_ptr) remote_query_executor->setMainTable(main_table); - pipes.emplace_back(createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes)); + remote_pipes.emplace_back(createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes)); }; const auto & settings = context.getSettingsRef(); @@ -275,7 +269,10 @@ void SelectStreamFactory::createForShard( } if (try_results.empty() || local_delay < max_remote_delay) - return createLocalPipe(modified_query_ast, header, context, stage); + { + auto plan = createLocalPlan(modified_query_ast, header, context, stage); + return QueryPipeline::getPipe(std::move(*plan->buildQueryPipeline())); + } else { std::vector connections; @@ -290,7 +287,7 @@ void SelectStreamFactory::createForShard( } }; - pipes.emplace_back(createDelayedPipe(header, lazily_create_stream, add_totals, add_extremes)); + delayed_pipes.emplace_back(createDelayedPipe(header, lazily_create_stream, add_totals, add_extremes)); } else emplace_remote_stream(); diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 80f72fd0024..9b57b92ed50 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -39,7 +39,9 @@ public: const String & query, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, const SelectQueryInfo & query_info, - Pipes & pipes) override; + std::vector & plans, + Pipes & remote_pipes, + Pipes & delayed_pipes) override; private: const Block header; diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 24d819ad2d7..6b4f28694a9 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include @@ -81,15 +84,19 @@ Context updateSettingsForCluster(const Cluster & cluster, const Context & contex return new_context; } -Pipe executeQuery( +void executeQuery( + QueryPlan & query_plan, IStreamFactory & stream_factory, Poco::Logger * log, const ASTPtr & query_ast, const Context & context, const SelectQueryInfo & query_info) { assert(log); - Pipes res; const Settings & settings = context.getSettingsRef(); + std::vector plans; + Pipes remote_pipes; + Pipes delayed_pipes; + const std::string query = queryToString(query_ast); Context new_context = updateSettingsForCluster(*query_info.cluster, context, settings, log); @@ -112,9 +119,43 @@ Pipe executeQuery( throttler = user_level_throttler; for (const auto & shard_info : query_info.cluster->getShardsInfo()) - stream_factory.createForShard(shard_info, query, query_ast, new_context, throttler, query_info, res); + stream_factory.createForShard(shard_info, query, query_ast, new_context, throttler, query_info, plans, remote_pipes, delayed_pipes); - return Pipe::unitePipes(std::move(res)); + if (!remote_pipes.empty()) + { + auto plan = std::make_unique(); + auto read_from_remote = std::make_unique(Pipe::unitePipes(std::move(remote_pipes))); + read_from_remote->setStepDescription("Read from remote replica"); + plan->addStep(std::move(read_from_remote)); + plans.emplace_back(std::move(plan)); + } + + if (!delayed_pipes.empty()) + { + auto plan = std::make_unique(); + auto read_from_remote = std::make_unique(Pipe::unitePipes(std::move(delayed_pipes))); + read_from_remote->setStepDescription("Read from delayed local replica"); + plan->addStep(std::move(read_from_remote)); + plans.emplace_back(std::move(plan)); + } + + if (plans.empty()) + return; + + if (plans.size() == 1) + { + query_plan = std::move(*plans.front()); + return; + } + + DataStreams input_streams; + input_streams.reserve(plans.size()); + for (auto & plan : plans) + input_streams.emplace_back(plan->getCurrentDataStream()); + + auto header = input_streams.front().header; + auto union_step = std::make_unique(std::move(input_streams), header); + query_plan.unitePlans(std::move(union_step), std::move(plans)); } } diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index e5d4aee2eb9..0b40c1412a1 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -11,6 +11,7 @@ class Cluster; struct SelectQueryInfo; class Pipe; +class QueryPlan; namespace ClusterProxy { @@ -31,8 +32,10 @@ Context updateSettingsForCluster(const Cluster & cluster, const Context & contex /// Execute a distributed query, creating a vector of BlockInputStreams, from which the result can be read. /// `stream_factory` object encapsulates the logic of creating streams for a different type of query /// (currently SELECT, DESCRIBE). -Pipe executeQuery( - IStreamFactory & stream_factory, Poco::Logger * log, const ASTPtr & query_ast, const Context & context, const SelectQueryInfo & query_info); +void executeQuery( + QueryPlan & query_plan, + IStreamFactory & stream_factory, Poco::Logger * log, + const ASTPtr & query_ast, const Context & context, const SelectQueryInfo & query_info); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 9b2339f930b..537dfca4d49 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -33,11 +33,13 @@ #include #include +#include #include #include #include +#include #include -#include +#include #include #include #include @@ -1108,6 +1110,48 @@ static StreamLocalLimits getLimitsForStorage(const Settings & settings, const Se return limits; } +void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, const Block & source_header, const SelectQueryInfo & query_info) +{ + Pipe pipe(std::make_shared(source_header)); + + if (query_info.prewhere_info) + { + if (query_info.prewhere_info->alias_actions) + { + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, query_info.prewhere_info->alias_actions); + }); + } + + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, + query_info.prewhere_info->prewhere_actions, + query_info.prewhere_info->prewhere_column_name, + query_info.prewhere_info->remove_prewhere_column); + }); + + // To remove additional columns + // In some cases, we did not read any marks so that the pipeline.streams is empty + // Thus, some columns in prewhere are not removed as expected + // This leads to mismatched header in distributed table + if (query_info.prewhere_info->remove_columns_actions) + { + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, query_info.prewhere_info->remove_columns_actions); + }); + } + } + + auto read_from_pipe = std::make_unique(std::move(pipe)); + read_from_pipe->setStepDescription("Read from NullSource"); + query_plan.addStep(std::move(read_from_pipe)); +} + void InterpreterSelectQuery::executeFetchColumns( QueryProcessingStage::Enum processing_stage, QueryPlan & query_plan, const PrewhereInfoPtr & prewhere_info, const Names & columns_to_remove_after_prewhere) @@ -1353,7 +1397,7 @@ void InterpreterSelectQuery::executeFetchColumns( ErrorCodes::TOO_MANY_COLUMNS); /// General limit for the number of threads. - query_plan.setMaxThreads(settings.max_threads); + size_t max_threads_execute_query = settings.max_threads; /** With distributed query processing, almost no computations are done in the threads, * but wait and receive data from remote servers. @@ -1366,8 +1410,7 @@ void InterpreterSelectQuery::executeFetchColumns( if (storage && storage->isRemote()) { is_remote = true; - max_streams = settings.max_distributed_connections; - query_plan.setMaxThreads(max_streams); + max_threads_execute_query = max_streams = settings.max_distributed_connections; } UInt64 max_block_size = settings.max_block_size; @@ -1392,8 +1435,7 @@ void InterpreterSelectQuery::executeFetchColumns( && limit_length + limit_offset < max_block_size) { max_block_size = std::max(UInt64(1), limit_length + limit_offset); - max_streams = 1; - query_plan.setMaxThreads(max_streams); + max_threads_execute_query = max_streams = 1; } if (!max_block_size) @@ -1473,12 +1515,36 @@ void InterpreterSelectQuery::executeFetchColumns( if (!options.ignore_quota && (options.to_stage == QueryProcessingStage::Complete)) quota = context->getQuota(); - storage->read(query_plan, table_lock, metadata_snapshot, limits, leaf_limits, std::move(quota), - required_columns, query_info, context, processing_stage, max_block_size, max_streams); + storage->read(query_plan, required_columns, metadata_snapshot, + query_info, *context, processing_stage, max_block_size, max_streams); + + /// Create step which reads from empty source if storage has no data. + if (!query_plan.isInitialized()) + { + auto header = metadata_snapshot->getSampleBlockForColumns( + required_columns, storage->getVirtuals(), storage->getStorageID()); + addEmptySourceToQueryPlan(query_plan, header, query_info); + } + + /// Extend lifetime of context, table lock, storage. Set limits and quota. + auto adding_limits_and_quota = std::make_unique( + query_plan.getCurrentDataStream(), + storage, + std::move(table_lock), + limits, + leaf_limits, + std::move(quota), + context); + adding_limits_and_quota->setStepDescription("Set limits and quota after reading from storage"); + query_plan.addStep(std::move(adding_limits_and_quota)); } else throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR); + /// Specify the number of threads only if it wasn't specified in storage. + if (!query_plan.getMaxThreads()) + query_plan.setMaxThreads(max_threads_execute_query); + /// Aliases in table declaration. if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions) { diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 455b1a1e623..5ed938af9f3 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -94,6 +94,8 @@ public: const SelectQueryInfo & getQueryInfo() const { return query_info; } + static void addEmptySourceToQueryPlan(QueryPlan & query_plan, const Block & source_header, const SelectQueryInfo & query_info); + private: InterpreterSelectQuery( const ASTPtr & query_ptr_, diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 3e7ebfec139..66cce64cff7 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ namespace DB namespace ErrorCodes { + extern const int NOT_IMPLEMENTED; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int UNKNOWN_MUTATION_COMMAND; @@ -92,6 +94,7 @@ std::optional findFirstNonDeterministicFunctionName(const MutationComman if (finder_data.nondeterministic_function_name) return finder_data.nondeterministic_function_name; + /// Currently UPDATE and DELETE both always have predicates so we can use fallthrough [[fallthrough]]; } @@ -110,7 +113,7 @@ std::optional findFirstNonDeterministicFunctionName(const MutationComman return {}; } -ASTPtr prepareQueryAffectedAST(const std::vector & commands) +ASTPtr prepareQueryAffectedAST(const std::vector & commands, const StoragePtr & storage, const Context & context) { /// Execute `SELECT count() FROM storage WHERE predicate1 OR predicate2 OR ...` query. /// The result can differ from the number of affected rows (e.g. if there is an UPDATE command that @@ -125,20 +128,23 @@ ASTPtr prepareQueryAffectedAST(const std::vector & commands) count_func->arguments = std::make_shared(); select->select()->children.push_back(count_func); - if (commands.size() == 1) - select->setExpression(ASTSelectQuery::Expression::WHERE, commands[0].predicate->clone()); - else + ASTs conditions; + for (const MutationCommand & command : commands) { - auto coalesced_predicates = std::make_shared(); - coalesced_predicates->name = "or"; - coalesced_predicates->arguments = std::make_shared(); - coalesced_predicates->children.push_back(coalesced_predicates->arguments); - - for (const MutationCommand & command : commands) - coalesced_predicates->arguments->children.push_back(command.predicate->clone()); + if (ASTPtr condition = getPartitionAndPredicateExpressionForMutationCommand(command, storage, context)) + conditions.push_back(std::move(condition)); + } + if (conditions.size() > 1) + { + auto coalesced_predicates = makeASTFunction("or"); + coalesced_predicates->arguments->children = std::move(conditions); select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(coalesced_predicates)); } + else if (conditions.size() == 1) + { + select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(conditions.front())); + } return select; } @@ -167,8 +173,9 @@ ColumnDependencies getAllColumnDependencies(const StorageMetadataPtr & metadata_ } + bool isStorageTouchedByMutations( - StoragePtr storage, + const StoragePtr & storage, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, Context context_copy) @@ -176,16 +183,33 @@ bool isStorageTouchedByMutations( if (commands.empty()) return false; + bool all_commands_can_be_skipped = true; + auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast(storage); for (const MutationCommand & command : commands) { if (!command.predicate) /// The command touches all rows. return true; + + if (command.partition && !storage_from_merge_tree_data_part) + throw Exception("ALTER UPDATE/DELETE ... IN PARTITION is not supported for non-MergeTree tables", ErrorCodes::NOT_IMPLEMENTED); + + if (command.partition && storage_from_merge_tree_data_part) + { + const String partition_id = storage_from_merge_tree_data_part->getPartitionIDFromQuery(command.partition, context_copy); + if (partition_id == storage_from_merge_tree_data_part->getPartitionId()) + all_commands_can_be_skipped = false; + } + else + all_commands_can_be_skipped = false; } + if (all_commands_can_be_skipped) + return false; + context_copy.setSetting("max_streams_to_max_threads_ratio", 1); context_copy.setSetting("max_threads", 1); - ASTPtr select_query = prepareQueryAffectedAST(commands); + ASTPtr select_query = prepareQueryAffectedAST(commands, storage, context_copy); /// Interpreter must be alive, when we use result of execute() method. /// For some reason it may copy context and and give it into ExpressionBlockInputStream @@ -202,9 +226,42 @@ bool isStorageTouchedByMutations( auto count = (*block.getByName("count()").column)[0].get(); return count != 0; - } + +ASTPtr getPartitionAndPredicateExpressionForMutationCommand( + const MutationCommand & command, + const StoragePtr & storage, + const Context & context +) +{ + ASTPtr partition_predicate_as_ast_func; + if (command.partition) + { + String partition_id; + + auto storage_merge_tree = std::dynamic_pointer_cast(storage); + auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast(storage); + if (storage_merge_tree) + partition_id = storage_merge_tree->getPartitionIDFromQuery(command.partition, context); + else if (storage_from_merge_tree_data_part) + partition_id = storage_from_merge_tree_data_part->getPartitionIDFromQuery(command.partition, context); + else + throw Exception("ALTER UPDATE/DELETE ... IN PARTITION is not supported for non-MergeTree tables", ErrorCodes::NOT_IMPLEMENTED); + + partition_predicate_as_ast_func = makeASTFunction("equals", + std::make_shared("_partition_id"), + std::make_shared(partition_id) + ); + } + + if (command.predicate && command.partition) + return makeASTFunction("and", command.predicate->clone(), std::move(partition_predicate_as_ast_func)); + else + return command.predicate ? command.predicate->clone() : partition_predicate_as_ast_func; +} + + MutationsInterpreter::MutationsInterpreter( StoragePtr storage_, const StorageMetadataPtr & metadata_snapshot_, @@ -349,7 +406,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) if (stages.empty() || !stages.back().column_to_updated.empty()) stages.emplace_back(context); - auto negated_predicate = makeASTFunction("isZeroOrNull", command.predicate->clone()); + auto negated_predicate = makeASTFunction("isZeroOrNull", getPartitionAndPredicateExpressionForMutationCommand(command)); stages.back().filters.push_back(negated_predicate); } else if (command.type == MutationCommand::UPDATE) @@ -387,7 +444,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) const auto & update_expr = kv.second; auto updated_column = makeASTFunction("CAST", makeASTFunction("if", - command.predicate->clone(), + getPartitionAndPredicateExpressionForMutationCommand(command), makeASTFunction("CAST", update_expr->clone(), type_literal), @@ -592,7 +649,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & for (const String & column : stage.output_columns) all_asts->children.push_back(std::make_shared(column)); - auto syntax_result = TreeRewriter(context).analyze(all_asts, all_columns); + auto syntax_result = TreeRewriter(context).analyze(all_asts, all_columns, storage, metadata_snapshot); if (context.hasQueryContext()) for (const auto & it : syntax_result->getScalars()) context.getQueryContext().addScalar(it.first, it.second); @@ -759,10 +816,10 @@ const Block & MutationsInterpreter::getUpdatedHeader() const size_t MutationsInterpreter::evaluateCommandsSize() { for (const MutationCommand & command : commands) - if (unlikely(!command.predicate)) /// The command touches all rows. + if (unlikely(!command.predicate && !command.partition)) /// The command touches all rows. return mutation_ast->size(); - return std::max(prepareQueryAffectedAST(commands)->size(), mutation_ast->size()); + return std::max(prepareQueryAffectedAST(commands, storage, context)->size(), mutation_ast->size()); } std::optional MutationsInterpreter::getStorageSortDescriptionIfPossible(const Block & header) const @@ -783,6 +840,11 @@ std::optional MutationsInterpreter::getStorageSortDescriptionIf return sort_description; } +ASTPtr MutationsInterpreter::getPartitionAndPredicateExpressionForMutationCommand(const MutationCommand & command) const +{ + return DB::getPartitionAndPredicateExpressionForMutationCommand(command, storage, context); +} + bool MutationsInterpreter::Stage::isAffectingAllColumns(const Names & storage_columns) const { /// is subset diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index 59d9e7657c3..18658e605ad 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -20,7 +20,17 @@ using QueryPipelinePtr = std::unique_ptr; /// Return false if the data isn't going to be changed by mutations. bool isStorageTouchedByMutations( - StoragePtr storage, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, Context context_copy); + const StoragePtr & storage, + const StorageMetadataPtr & metadata_snapshot, + const std::vector & commands, + Context context_copy +); + +ASTPtr getPartitionAndPredicateExpressionForMutationCommand( + const MutationCommand & command, + const StoragePtr & storage, + const Context & context +); /// Create an input stream that will read data from storage and apply mutation commands (UPDATEs, DELETEs, MATERIALIZEs) /// to this data. @@ -59,6 +69,8 @@ private: std::optional getStorageSortDescriptionIfPossible(const Block & header) const; + ASTPtr getPartitionAndPredicateExpressionForMutationCommand(const MutationCommand & command) const; + StoragePtr storage; StorageMetadataPtr metadata_snapshot; MutationCommands commands; diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 1e396a0fed1..01077602d90 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -25,6 +25,7 @@ namespace { constexpr size_t DEFAULT_SYSTEM_LOG_FLUSH_INTERVAL_MILLISECONDS = 7500; +constexpr size_t DEFAULT_METRIC_LOG_COLLECT_INTERVAL_MILLISECONDS = 1000; /// Creates a system log with MergeTree engine using parameters from config template @@ -125,7 +126,8 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi if (metric_log) { - size_t collect_interval_milliseconds = config.getUInt64("metric_log.collect_interval_milliseconds"); + size_t collect_interval_milliseconds = config.getUInt64("metric_log.collect_interval_milliseconds", + DEFAULT_METRIC_LOG_COLLECT_INTERVAL_MILLISECONDS); metric_log->startCollectMetric(collect_interval_milliseconds); } diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 8d3cb123955..51609cbaa23 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -63,7 +63,7 @@ struct CustomizeFunctionsData const String & customized_func_name; - void visit(ASTFunction & func, ASTPtr &) + void visit(ASTFunction & func, ASTPtr &) const { if (Poco::toLower(func.name) == func_name) { @@ -97,7 +97,7 @@ struct CustomizeFunctionsSuffixData const String & customized_func_suffix; - void visit(ASTFunction & func, ASTPtr &) + void visit(ASTFunction & func, ASTPtr &) const { if (endsWith(Poco::toLower(func.name), func_suffix)) { @@ -118,7 +118,7 @@ struct CustomizeAggregateFunctionsSuffixData const String & customized_func_suffix; - void visit(ASTFunction & func, ASTPtr &) + void visit(ASTFunction & func, ASTPtr &) const { const auto & instance = AggregateFunctionFactory::instance(); if (instance.isAggregateFunctionName(func.name) && !endsWith(func.name, customized_func_suffix)) diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 918bd4acaa7..d07e57eefae 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -90,7 +90,7 @@ void ASTAlterCommand::formatImpl( column->formatImpl(settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } } @@ -150,7 +150,7 @@ void ASTAlterCommand::formatImpl( index->formatImpl(settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } } @@ -161,7 +161,7 @@ void ASTAlterCommand::formatImpl( index->formatImpl(settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } } @@ -272,7 +272,15 @@ void ASTAlterCommand::formatImpl( } else if (type == ASTAlterCommand::DELETE) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DELETE WHERE " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DELETE" << (settings.hilite ? hilite_none : ""); + + if (partition) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(settings, state, frame); + } + + settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); predicate->formatImpl(settings, state, frame); } else if (type == ASTAlterCommand::UPDATE) @@ -280,6 +288,12 @@ void ASTAlterCommand::formatImpl( settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "UPDATE " << (settings.hilite ? hilite_none : ""); update_assignments->formatImpl(settings, state, frame); + if (partition) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(settings, state, frame); + } + settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); predicate->formatImpl(settings, state, frame); } @@ -298,7 +312,7 @@ void ASTAlterCommand::formatImpl( << (settings.hilite ? hilite_none : ""); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } } diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 78e0c726ddf..65657e5ecfd 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -103,7 +103,7 @@ public: */ ASTPtr constraint; - /** Used in DROP PARTITION and ATTACH PARTITION FROM queries. + /** Used in DROP PARTITION, ATTACH PARTITION FROM, UPDATE, DELETE queries. * The value or ID of the partition is stored here. */ ASTPtr partition; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index ad03d949174..0f06a0d2480 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -55,6 +55,12 @@ const char * ParserComparisonExpression::operators[] = nullptr }; +const char * ParserComparisonExpression::overlapping_operators_to_skip[] = +{ + "IN PARTITION", + nullptr +}; + const char * ParserLogicalNotExpression::operators[] = { "NOT", "not", @@ -137,6 +143,14 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node /// try to find any of the valid operators const char ** it; + Expected stub; + for (it = overlapping_operators_to_skip; *it; ++it) + if (ParserKeyword{*it}.checkWithoutMoving(pos, stub)) + break; + + if (*it) + break; + for (it = operators; *it; it += 2) if (parseOperator(pos, *it, expected)) break; diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index cf77b8b4da4..4e21eff7f0e 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -82,6 +82,7 @@ class ParserLeftAssociativeBinaryOperatorList : public IParserBase { private: Operators_t operators; + Operators_t overlapping_operators_to_skip = { (const char *[]){ nullptr } }; ParserPtr first_elem_parser; ParserPtr remaining_elem_parser; @@ -93,6 +94,11 @@ public: { } + ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, Operators_t overlapping_operators_to_skip_, ParserPtr && first_elem_parser_) + : operators(operators_), overlapping_operators_to_skip(overlapping_operators_to_skip_), first_elem_parser(std::move(first_elem_parser_)) + { + } + ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, ParserPtr && first_elem_parser_, ParserPtr && remaining_elem_parser_) : operators(operators_), first_elem_parser(std::move(first_elem_parser_)), @@ -284,7 +290,8 @@ class ParserComparisonExpression : public IParserBase { private: static const char * operators[]; - ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique()}; + static const char * overlapping_operators_to_skip[]; + ParserLeftAssociativeBinaryOperatorList operator_parser {operators, overlapping_operators_to_skip, std::make_unique()}; protected: const char * getName() const override{ return "comparison expression"; } diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index bb682455a99..7050614007e 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -79,7 +79,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_to_volume("TO VOLUME"); ParserKeyword s_to_table("TO TABLE"); - ParserKeyword s_delete_where("DELETE WHERE"); + ParserKeyword s_delete("DELETE"); ParserKeyword s_update("UPDATE"); ParserKeyword s_where("WHERE"); ParserKeyword s_to("TO"); @@ -506,8 +506,17 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->type = ASTAlterCommand::MODIFY_SAMPLE_BY; } - else if (s_delete_where.ignore(pos, expected)) + else if (s_delete.ignore(pos, expected)) { + if (s_in_partition.ignore(pos, expected)) + { + if (!parser_partition.parse(pos, command->partition, expected)) + return false; + } + + if (!s_where.ignore(pos, expected)) + return false; + if (!parser_exp_elem.parse(pos, command->predicate, expected)) return false; @@ -518,6 +527,12 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected if (!parser_assignment_list.parse(pos, command->update_assignments, expected)) return false; + if (s_in_partition.ignore(pos, expected)) + { + if (!parser_partition.parse(pos, command->partition, expected)) + return false; + } + if (!s_where.ignore(pos, expected)) return false; diff --git a/src/Parsers/ParserAlterQuery.h b/src/Parsers/ParserAlterQuery.h index a0981c77ca6..514ef876430 100644 --- a/src/Parsers/ParserAlterQuery.h +++ b/src/Parsers/ParserAlterQuery.h @@ -10,7 +10,7 @@ namespace DB * ALTER TABLE [db.]name [ON CLUSTER cluster] * [ADD COLUMN [IF NOT EXISTS] col_name type [AFTER col_after],] * [DROP COLUMN [IF EXISTS] col_to_drop, ...] - * [CLEAR COLUMN [IF EXISTS] col_to_clear [IN PARTITION partition],] + * [CLEAR COLUMN [IF EXISTS] col_to_clear[ IN PARTITION partition],] * [MODIFY COLUMN [IF EXISTS] col_to_modify type, ...] * [RENAME COLUMN [IF EXISTS] col_name TO col_name] * [MODIFY PRIMARY KEY (a, b, c...)] @@ -19,8 +19,12 @@ namespace DB * [DROP|DETACH|ATTACH PARTITION|PART partition, ...] * [FETCH PARTITION partition FROM ...] * [FREEZE [PARTITION] [WITH NAME name]] - * [DELETE WHERE ...] - * [UPDATE col_name = expr, ... WHERE ...] + * [DELETE[ IN PARTITION partition] WHERE ...] + * [UPDATE col_name = expr, ...[ IN PARTITION partition] WHERE ...] + * [ADD INDEX [IF NOT EXISTS] index_name [AFTER index_name]] + * [DROP INDEX [IF EXISTS] index_name] + * [CLEAR INDEX [IF EXISTS] index_name IN PARTITION partition] + * [MATERIALIZE INDEX [IF EXISTS] index_name [IN PARTITION partition]] * ALTER LIVE VIEW [db.name] * [REFRESH] */ diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index b679f12ddfe..516a9a77b71 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -95,6 +95,12 @@ void QueryPipeline::addTransform(ProcessorPtr transform) pipe.addTransform(std::move(transform)); } +void QueryPipeline::transform(const Transformer & transformer) +{ + checkInitializedAndNotCompleted(); + pipe.transform(transformer); +} + void QueryPipeline::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter) { checkInitializedAndNotCompleted(); diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 2ef0fc7dadd..c8da3e91291 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -53,6 +53,11 @@ public: void addSimpleTransform(const Pipe::ProcessorGetterWithStreamKind & getter); /// Add transform with getNumStreams() input ports. void addTransform(ProcessorPtr transform); + + using Transformer = std::function; + /// Transform pipeline in general way. + void transform(const Transformer & transformer); + /// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port. void addTotalsHavingTransform(ProcessorPtr transform); /// Add transform which calculates extremes. This transform adds extremes port and doesn't change inputs number. @@ -105,6 +110,9 @@ public: void addInterpreterContext(std::shared_ptr context) { pipe.addInterpreterContext(std::move(context)); } void addStorageHolder(StoragePtr storage) { pipe.addStorageHolder(std::move(storage)); } void addQueryPlan(std::unique_ptr plan) { pipe.addQueryPlan(std::move(plan)); } + void setLimits(const StreamLocalLimits & limits) { pipe.setLimits(limits); } + void setLeafLimits(const SizeLimits & limits) { pipe.setLeafLimits(limits); } + void setQuota(const std::shared_ptr & quota) { pipe.setQuota(quota); } /// For compatibility with IBlockInputStream. void setProgressCallback(const ProgressCallback & callback); diff --git a/src/Processors/QueryPlan/AddingConstColumnStep.cpp b/src/Processors/QueryPlan/AddingConstColumnStep.cpp new file mode 100644 index 00000000000..27c7720e58e --- /dev/null +++ b/src/Processors/QueryPlan/AddingConstColumnStep.cpp @@ -0,0 +1,41 @@ +#include +#include +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +AddingConstColumnStep::AddingConstColumnStep(const DataStream & input_stream_, ColumnWithTypeAndName column_) + : ITransformingStep(input_stream_, + AddingConstColumnTransform::transformHeader(input_stream_.header, column_), + getTraits()) + , column(std::move(column_)) +{ +} + +void AddingConstColumnStep::transformPipeline(QueryPipeline & pipeline) +{ + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, column); + }); +} + +} diff --git a/src/Processors/QueryPlan/AddingConstColumnStep.h b/src/Processors/QueryPlan/AddingConstColumnStep.h new file mode 100644 index 00000000000..baa63873f21 --- /dev/null +++ b/src/Processors/QueryPlan/AddingConstColumnStep.h @@ -0,0 +1,22 @@ +#pragma once +#include + +namespace DB +{ + +/// Adds a materialized const column with a specified value. +class AddingConstColumnStep : public ITransformingStep +{ +public: + AddingConstColumnStep(const DataStream & input_stream_, ColumnWithTypeAndName column_); + + String getName() const override { return "AddingConstColumn"; } + + void transformPipeline(QueryPipeline & pipeline) override; + +private: + ColumnWithTypeAndName column; +}; + +} + diff --git a/src/Processors/QueryPlan/AddingMissedStep.cpp b/src/Processors/QueryPlan/AddingMissedStep.cpp new file mode 100644 index 00000000000..d354405b5ac --- /dev/null +++ b/src/Processors/QueryPlan/AddingMissedStep.cpp @@ -0,0 +1,45 @@ +#include +#include +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +AddingMissedStep::AddingMissedStep( + const DataStream & input_stream_, + Block result_header_, + ColumnsDescription columns_, + const Context & context_) + : ITransformingStep(input_stream_, result_header_, getTraits()) + , columns(std::move(columns_)) + , context(context_) +{ + updateDistinctColumns(output_stream->header, output_stream->distinct_columns); +} + +void AddingMissedStep::transformPipeline(QueryPipeline & pipeline) +{ + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, output_stream->header, columns, context); + }); +} + +} diff --git a/src/Processors/QueryPlan/AddingMissedStep.h b/src/Processors/QueryPlan/AddingMissedStep.h new file mode 100644 index 00000000000..ce755b79fdf --- /dev/null +++ b/src/Processors/QueryPlan/AddingMissedStep.h @@ -0,0 +1,26 @@ +#pragma once +#include +#include + +namespace DB +{ + +/// Convert one block structure to another. See ConvertingTransform. +class AddingMissedStep : public ITransformingStep +{ +public: + AddingMissedStep(const DataStream & input_stream_, + Block result_header_, + ColumnsDescription columns_, + const Context & context_); + + String getName() const override { return "AddingMissed"; } + + void transformPipeline(QueryPipeline & pipeline) override; + +private: + ColumnsDescription columns; + const Context & context; +}; + +} diff --git a/src/Processors/QueryPlan/ISourceStep.cpp b/src/Processors/QueryPlan/ISourceStep.cpp index cf68104f18c..1796f033896 100644 --- a/src/Processors/QueryPlan/ISourceStep.cpp +++ b/src/Processors/QueryPlan/ISourceStep.cpp @@ -14,7 +14,8 @@ QueryPipelinePtr ISourceStep::updatePipeline(QueryPipelines) auto pipeline = std::make_unique(); QueryPipelineProcessorsCollector collector(*pipeline, this); initializePipeline(*pipeline); - processors = collector.detachProcessors(); + auto added_processors = collector.detachProcessors(); + processors.insert(processors.end(), added_processors.begin(), added_processors.end()); return pipeline; } diff --git a/src/Processors/QueryPlan/ISourceStep.h b/src/Processors/QueryPlan/ISourceStep.h index 54bc19957f4..fdb3dd566cb 100644 --- a/src/Processors/QueryPlan/ISourceStep.h +++ b/src/Processors/QueryPlan/ISourceStep.h @@ -16,7 +16,7 @@ public: void describePipeline(FormatSettings & settings) const override; -private: +protected: /// We collect processors got after pipeline transformation. Processors processors; }; diff --git a/src/Processors/QueryPlan/MaterializingStep.cpp b/src/Processors/QueryPlan/MaterializingStep.cpp new file mode 100644 index 00000000000..f5313369020 --- /dev/null +++ b/src/Processors/QueryPlan/MaterializingStep.cpp @@ -0,0 +1,39 @@ +#include +#include +#include + +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +MaterializingStep::MaterializingStep(const DataStream & input_stream_) + : ITransformingStep(input_stream_, materializeBlock(input_stream_.header), getTraits()) +{ +} + +void MaterializingStep::transformPipeline(QueryPipeline & pipeline) +{ + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header); + }); +} + +} diff --git a/src/Processors/QueryPlan/MaterializingStep.h b/src/Processors/QueryPlan/MaterializingStep.h new file mode 100644 index 00000000000..72b3133dfe4 --- /dev/null +++ b/src/Processors/QueryPlan/MaterializingStep.h @@ -0,0 +1,18 @@ +#pragma once +#include + +namespace DB +{ + +/// Materialize constants. See MaterializingTransform. +class MaterializingStep : public ITransformingStep +{ +public: + explicit MaterializingStep(const DataStream & input_stream_); + + String getName() const override { return "Materializing"; } + + void transformPipeline(QueryPipeline & pipeline) override; +}; + +} diff --git a/src/Processors/QueryPlan/MergingFinal.cpp b/src/Processors/QueryPlan/MergingFinal.cpp new file mode 100644 index 00000000000..c1dec231f11 --- /dev/null +++ b/src/Processors/QueryPlan/MergingFinal.cpp @@ -0,0 +1,164 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = false, + .preserves_sorting = false, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +MergingFinal::MergingFinal( + const DataStream & input_stream, + size_t num_output_streams_, + SortDescription sort_description_, + MergeTreeData::MergingParams params_, + Names partition_key_columns_, + size_t max_block_size_) + : ITransformingStep(input_stream, input_stream.header, getTraits()) + , num_output_streams(num_output_streams_) + , sort_description(std::move(sort_description_)) + , merging_params(std::move(params_)) + , partition_key_columns(std::move(partition_key_columns_)) + , max_block_size(max_block_size_) +{ + /// TODO: check input_stream is partially sorted (each port) by the same description. +// output_stream->sort_description = sort_description; +// output_stream->sort_mode = DataStream::SortMode::Stream; +} + +void MergingFinal::transformPipeline(QueryPipeline & pipeline) +{ + const auto & header = pipeline.getHeader(); + size_t num_outputs = pipeline.getNumStreams(); + + auto get_merging_processor = [&]() -> MergingTransformPtr + { + switch (merging_params.mode) + { + case MergeTreeData::MergingParams::Ordinary: + { + return std::make_shared(header, num_outputs, + sort_description, max_block_size); + } + + case MergeTreeData::MergingParams::Collapsing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.sign_column, true, max_block_size); + + case MergeTreeData::MergingParams::Summing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.columns_to_sum, partition_key_columns, max_block_size); + + case MergeTreeData::MergingParams::Aggregating: + return std::make_shared(header, num_outputs, + sort_description, max_block_size); + + case MergeTreeData::MergingParams::Replacing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.version_column, max_block_size); + + case MergeTreeData::MergingParams::VersionedCollapsing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.sign_column, max_block_size); + + case MergeTreeData::MergingParams::Graphite: + throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR); + } + + __builtin_unreachable(); + }; + + if (num_output_streams <= 1 || sort_description.empty()) + { + pipeline.addTransform(get_merging_processor()); + return; + } + + ColumnNumbers key_columns; + key_columns.reserve(sort_description.size()); + + for (auto & desc : sort_description) + { + if (!desc.column_name.empty()) + key_columns.push_back(header.getPositionByName(desc.column_name)); + else + key_columns.emplace_back(desc.column_number); + } + + pipeline.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, num_output_streams, key_columns); + }); + + pipeline.transform([&](OutputPortRawPtrs ports) + { + Processors transforms; + std::vector output_ports; + transforms.reserve(ports.size() + num_output_streams); + output_ports.reserve(ports.size()); + + for (auto & port : ports) + { + auto copier = std::make_shared(header, num_output_streams); + connect(*port, copier->getInputPort()); + output_ports.emplace_back(copier->getOutputs().begin()); + transforms.emplace_back(std::move(copier)); + } + + for (size_t i = 0; i < num_output_streams; ++i) + { + auto merge = get_merging_processor(); + merge->setSelectorPosition(i); + auto input = merge->getInputs().begin(); + + /// Connect i-th merge with i-th input port of every copier. + for (size_t j = 0; j < ports.size(); ++j) + { + connect(*output_ports[j], *input); + ++output_ports[j]; + ++input; + } + + transforms.emplace_back(std::move(merge)); + } + + return transforms; + }); +} + +void MergingFinal::describeActions(FormatSettings & settings) const +{ + String prefix(settings.offset, ' '); + settings.out << prefix << "Sort description: "; + dumpSortDescription(sort_description, input_streams.front().header, settings.out); + settings.out << '\n'; +} + +} diff --git a/src/Processors/QueryPlan/MergingFinal.h b/src/Processors/QueryPlan/MergingFinal.h new file mode 100644 index 00000000000..c01f5c7f9a1 --- /dev/null +++ b/src/Processors/QueryPlan/MergingFinal.h @@ -0,0 +1,35 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +/// Merge streams of data into single sorted stream. +class MergingFinal : public ITransformingStep +{ +public: + explicit MergingFinal( + const DataStream & input_stream, + size_t num_output_streams_, + SortDescription sort_description_, + MergeTreeData::MergingParams params_, + Names partition_key_columns_, + size_t max_block_size_); + + String getName() const override { return "MergingFinal"; } + + void transformPipeline(QueryPipeline & pipeline) override; + + void describeActions(FormatSettings & settings) const override; + +private: + size_t num_output_streams; + SortDescription sort_description; + MergeTreeData::MergingParams merging_params; + Names partition_key_columns; + size_t max_block_size; +}; + +} diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index 1ff844480a9..e681d4989d3 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -185,6 +185,17 @@ QueryPipelinePtr QueryPlan::buildQueryPipeline() return last_pipeline; } +Pipe QueryPlan::convertToPipe() +{ + if (!isInitialized()) + return {}; + + if (isCompleted()) + throw Exception("Cannot convert completed QueryPlan to Pipe", ErrorCodes::LOGICAL_ERROR); + + return QueryPipeline::getPipe(std::move(*buildQueryPipeline())); +} + void QueryPlan::addInterpreterContext(std::shared_ptr context) { interpreter_context.emplace_back(std::move(context)); diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 6296eac7502..d53193a7aa5 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -17,6 +17,11 @@ using QueryPipelinePtr = std::unique_ptr; class Context; class WriteBuffer; +class QueryPlan; +using QueryPlanPtr = std::unique_ptr; + +class Pipe; + /// A tree of query steps. /// The goal of QueryPlan is to build QueryPipeline. /// QueryPlan let delay pipeline creation which is helpful for pipeline-level optimisations. @@ -28,7 +33,7 @@ public: QueryPlan(QueryPlan &&); QueryPlan & operator=(QueryPlan &&); - void unitePlans(QueryPlanStepPtr step, std::vector> plans); + void unitePlans(QueryPlanStepPtr step, std::vector plans); void addStep(QueryPlanStepPtr step); bool isInitialized() const { return root != nullptr; } /// Tree is not empty @@ -39,6 +44,9 @@ public: QueryPipelinePtr buildQueryPipeline(); + /// If initialized, build pipeline and convert to pipe. Otherwise, return empty pipe. + Pipe convertToPipe(); + struct ExplainPlanOptions { /// Add output header to step. @@ -61,6 +69,7 @@ public: /// Set upper limit for the recommend number of threads. Will be applied to the newly-created pipelines. /// TODO: make it in a better way. void setMaxThreads(size_t max_threads_) { max_threads = max_threads_; } + size_t getMaxThreads() const { return max_threads; } void addInterpreterContext(std::shared_ptr context); diff --git a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp index 979b4101046..2d4b759b637 100644 --- a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp +++ b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp @@ -5,7 +5,7 @@ namespace DB { ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_) - : ISourceStep(DataStream{.header = pipe_.getHeader(), .has_single_port = true}) + : ISourceStep(DataStream{.header = pipe_.getHeader()}) , pipe(std::move(pipe_)) , context(std::move(context_)) { @@ -13,7 +13,11 @@ ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_ = nullptr); - String getName() const override { return "ReadNothing"; } + String getName() const override { return "ReadFromPreparedSource"; } void initializePipeline(QueryPipeline & pipeline) override; @@ -20,4 +20,16 @@ private: std::shared_ptr context; }; +class ReadFromStorageStep : public ReadFromPreparedSource +{ +public: + ReadFromStorageStep(Pipe pipe_, String storage_name) + : ReadFromPreparedSource(std::move(pipe_)) + { + setStepDescription(storage_name); + } + + String getName() const override { return "ReadFromStorage"; } +}; + } diff --git a/src/Processors/QueryPlan/ReadFromStorageStep.cpp b/src/Processors/QueryPlan/ReadFromStorageStep.cpp deleted file mode 100644 index dfc2151f836..00000000000 --- a/src/Processors/QueryPlan/ReadFromStorageStep.cpp +++ /dev/null @@ -1,114 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -ReadFromStorageStep::ReadFromStorageStep( - TableLockHolder table_lock, - StorageMetadataPtr metadata_snapshot, - StreamLocalLimits & limits, - SizeLimits & leaf_limits, - std::shared_ptr quota, - StoragePtr storage, - const Names & required_columns, - SelectQueryInfo & query_info, - std::shared_ptr context, - QueryProcessingStage::Enum processing_stage, - size_t max_block_size, - size_t max_streams) -{ - /// Note: we read from storage in constructor of step because we don't know real header before reading. - /// It will be fixed when storage return QueryPlanStep itself. - - Pipe pipe = storage->read(required_columns, metadata_snapshot, query_info, *context, processing_stage, max_block_size, max_streams); - - if (pipe.empty()) - { - pipe = Pipe(std::make_shared(metadata_snapshot->getSampleBlockForColumns(required_columns, storage->getVirtuals(), storage->getStorageID()))); - - if (query_info.prewhere_info) - { - if (query_info.prewhere_info->alias_actions) - { - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, query_info.prewhere_info->alias_actions); - }); - } - - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, - query_info.prewhere_info->prewhere_actions, - query_info.prewhere_info->prewhere_column_name, - query_info.prewhere_info->remove_prewhere_column); - }); - - // To remove additional columns - // In some cases, we did not read any marks so that the pipeline.streams is empty - // Thus, some columns in prewhere are not removed as expected - // This leads to mismatched header in distributed table - if (query_info.prewhere_info->remove_columns_actions) - { - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, query_info.prewhere_info->remove_columns_actions); - }); - } - } - } - - pipeline = std::make_unique(); - QueryPipelineProcessorsCollector collector(*pipeline, this); - - pipe.setLimits(limits); - - /** - * Leaf size limits should be applied only for local processing of distributed queries. - * Such limits allow to control the read stage on leaf nodes and exclude the merging stage. - * Consider the case when distributed query needs to read from multiple shards. Then leaf - * limits will be applied on the shards only (including the root node) but will be ignored - * on the results merging stage. - */ - if (!storage->isRemote()) - pipe.setLeafLimits(leaf_limits); - - if (quota) - pipe.setQuota(quota); - - pipeline->init(std::move(pipe)); - - /// Add resources to pipeline. The order is important. - /// Add in reverse order of destruction. Pipeline will be destroyed at the end in case of exception. - pipeline->addInterpreterContext(std::move(context)); - pipeline->addStorageHolder(std::move(storage)); - pipeline->addTableLock(std::move(table_lock)); - - processors = collector.detachProcessors(); - - output_stream = DataStream{.header = pipeline->getHeader(), .has_single_port = pipeline->getNumStreams() == 1}; -} - -ReadFromStorageStep::~ReadFromStorageStep() = default; - -QueryPipelinePtr ReadFromStorageStep::updatePipeline(QueryPipelines) -{ - return std::move(pipeline); -} - -void ReadFromStorageStep::describePipeline(FormatSettings & settings) const -{ - IQueryPlanStep::describePipeline(processors, settings); -} - -} diff --git a/src/Processors/QueryPlan/ReadFromStorageStep.h b/src/Processors/QueryPlan/ReadFromStorageStep.h deleted file mode 100644 index c803d8af63f..00000000000 --- a/src/Processors/QueryPlan/ReadFromStorageStep.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once -#include -#include -#include -#include - -namespace DB -{ - -class IStorage; -using StoragePtr = std::shared_ptr; - -struct StorageInMemoryMetadata; -using StorageMetadataPtr = std::shared_ptr; - -struct SelectQueryInfo; - -struct PrewhereInfo; - -class EnabledQuota; - -/// Reads from storage. -class ReadFromStorageStep : public IQueryPlanStep -{ -public: - ReadFromStorageStep( - TableLockHolder table_lock, - StorageMetadataPtr metadata_snapshot, - StreamLocalLimits & limits, - SizeLimits & leaf_limits, - std::shared_ptr quota, - StoragePtr storage, - const Names & required_columns, - SelectQueryInfo & query_info, - std::shared_ptr context, - QueryProcessingStage::Enum processing_stage, - size_t max_block_size, - size_t max_streams); - - ~ReadFromStorageStep() override; - - String getName() const override { return "ReadFromStorage"; } - - QueryPipelinePtr updatePipeline(QueryPipelines) override; - - void describePipeline(FormatSettings & settings) const override; - -private: - QueryPipelinePtr pipeline; - Processors processors; -}; - -} diff --git a/src/Processors/QueryPlan/ReverseRowsStep.cpp b/src/Processors/QueryPlan/ReverseRowsStep.cpp new file mode 100644 index 00000000000..32e16937611 --- /dev/null +++ b/src/Processors/QueryPlan/ReverseRowsStep.cpp @@ -0,0 +1,37 @@ +#include +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = false, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +ReverseRowsStep::ReverseRowsStep(const DataStream & input_stream_) + : ITransformingStep(input_stream_, input_stream_.header, getTraits()) +{ +} + +void ReverseRowsStep::transformPipeline(QueryPipeline & pipeline) +{ + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header); + }); +} + +} diff --git a/src/Processors/QueryPlan/ReverseRowsStep.h b/src/Processors/QueryPlan/ReverseRowsStep.h new file mode 100644 index 00000000000..955d022cde0 --- /dev/null +++ b/src/Processors/QueryPlan/ReverseRowsStep.h @@ -0,0 +1,18 @@ +#pragma once +#include + +namespace DB +{ + +/// Reverse rows in chunk. +class ReverseRowsStep : public ITransformingStep +{ +public: + ReverseRowsStep(const DataStream & input_stream_); + + String getName() const override { return "ReverseRows"; } + + void transformPipeline(QueryPipeline & pipeline) override; +}; + +} diff --git a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp new file mode 100644 index 00000000000..588dd7599a1 --- /dev/null +++ b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp @@ -0,0 +1,71 @@ +#include +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +SettingQuotaAndLimitsStep::SettingQuotaAndLimitsStep( + const DataStream & input_stream_, + StoragePtr storage_, + TableLockHolder table_lock_, + StreamLocalLimits & limits_, + SizeLimits & leaf_limits_, + std::shared_ptr quota_, + std::shared_ptr context_) + : ITransformingStep(input_stream_, input_stream_.header, getTraits()) + , context(std::move(context_)) + , storage(std::move(storage_)) + , table_lock(std::move(table_lock_)) + , limits(limits_) + , leaf_limits(leaf_limits_) + , quota(std::move(quota_)) +{ +} + +void SettingQuotaAndLimitsStep::transformPipeline(QueryPipeline & pipeline) +{ + /// Table lock is stored inside pipeline here. + pipeline.setLimits(limits); + + /** + * Leaf size limits should be applied only for local processing of distributed queries. + * Such limits allow to control the read stage on leaf nodes and exclude the merging stage. + * Consider the case when distributed query needs to read from multiple shards. Then leaf + * limits will be applied on the shards only (including the root node) but will be ignored + * on the results merging stage. + */ + if (!storage->isRemote()) + pipeline.setLeafLimits(leaf_limits); + + if (quota) + pipeline.setQuota(quota); + + /// Order of resources below is important. + if (context) + pipeline.addInterpreterContext(std::move(context)); + + if (storage) + pipeline.addStorageHolder(std::move(storage)); + + if (table_lock) + pipeline.addTableLock(std::move(table_lock)); +} + +} diff --git a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h new file mode 100644 index 00000000000..66e44e18cd4 --- /dev/null +++ b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h @@ -0,0 +1,44 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +class IStorage; +using StoragePtr = std::shared_ptr; + +struct StorageInMemoryMetadata; +using StorageMetadataPtr = std::shared_ptr; + +class EnabledQuota; + +/// Add limits, quota, table_lock and other stuff to pipeline. +/// Doesn't change DataStream. +class SettingQuotaAndLimitsStep : public ITransformingStep +{ +public: + SettingQuotaAndLimitsStep( + const DataStream & input_stream_, + StoragePtr storage_, + TableLockHolder table_lock_, + StreamLocalLimits & limits_, + SizeLimits & leaf_limits_, + std::shared_ptr quota_, + std::shared_ptr context_); + + String getName() const override { return "SettingQuotaAndLimits"; } + + void transformPipeline(QueryPipeline & pipeline) override; + +private: + std::shared_ptr context; + StoragePtr storage; + TableLockHolder table_lock; + StreamLocalLimits limits; + SizeLimits leaf_limits; + std::shared_ptr quota; +}; + +} diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index 1e74046b071..630ff53f47d 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -30,7 +30,7 @@ QueryPipelinePtr UnionStep::updatePipeline(QueryPipelines pipelines) return pipeline; } - *pipeline = QueryPipeline::unitePipelines(std::move(pipelines), output_stream->header ,max_threads); + *pipeline = QueryPipeline::unitePipelines(std::move(pipelines), output_stream->header, max_threads); processors = collector.detachProcessors(); return pipeline; diff --git a/src/Processors/QueryPlan/UnionStep.h b/src/Processors/QueryPlan/UnionStep.h index 9e00e24279b..e2e1f2c9efa 100644 --- a/src/Processors/QueryPlan/UnionStep.h +++ b/src/Processors/QueryPlan/UnionStep.h @@ -9,7 +9,7 @@ class UnionStep : public IQueryPlanStep { public: /// max_threads is used to limit the number of threads for result pipeline. - UnionStep(DataStreams input_streams_, Block result_header, size_t max_threads_); + UnionStep(DataStreams input_streams_, Block result_header, size_t max_threads_ = 0); String getName() const override { return "Union"; } diff --git a/src/Processors/Transforms/AddingConstColumnTransform.h b/src/Processors/Transforms/AddingConstColumnTransform.h index 26d70d27ca7..15e9addd924 100644 --- a/src/Processors/Transforms/AddingConstColumnTransform.h +++ b/src/Processors/Transforms/AddingConstColumnTransform.h @@ -4,33 +4,40 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + /// Adds a materialized const column to the chunk with a specified value. -template class AddingConstColumnTransform : public ISimpleTransform { public: - AddingConstColumnTransform(const Block & header, DataTypePtr data_type_, T value_, const String & column_name_) - : ISimpleTransform(header, addColumn(header, data_type_, column_name_), false) - , data_type(std::move(data_type_)), value(value_) {} + AddingConstColumnTransform(const Block & header, ColumnWithTypeAndName column_) + : ISimpleTransform(header, transformHeader(header, column_), false) + , column(std::move(column_)) + { + if (!column.column || !isColumnConst(*column.column) || !column.column->empty()) + throw Exception("AddingConstColumnTransform expected empty const column", ErrorCodes::LOGICAL_ERROR); + } String getName() const override { return "AddingConstColumnTransform"; } + static Block transformHeader(Block header, ColumnWithTypeAndName & column_) + { + header.insert(column_); + return header; + } + protected: void transform(Chunk & chunk) override { auto num_rows = chunk.getNumRows(); - chunk.addColumn(data_type->createColumnConst(num_rows, value)->convertToFullColumnIfConst()); + chunk.addColumn(column.column->cloneResized(num_rows)->convertToFullColumnIfConst()); } private: - static Block addColumn(Block header, const DataTypePtr & data_type, const String & column_name) - { - header.insert({data_type->createColumn(), data_type, column_name}); - return header; - } - - DataTypePtr data_type; - T value; + ColumnWithTypeAndName column; }; } diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 9f022d26d3a..5c9de498e5a 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -88,7 +88,9 @@ SRCS( Pipe.cpp Port.cpp QueryPipeline.cpp + QueryPlan/AddingConstColumnStep.cpp QueryPlan/AddingDelayedSourceStep.cpp + QueryPlan/AddingMissedStep.cpp QueryPlan/AggregatingStep.cpp QueryPlan/ArrayJoinStep.cpp QueryPlan/ConvertingStep.cpp @@ -105,16 +107,19 @@ SRCS( QueryPlan/ITransformingStep.cpp QueryPlan/LimitByStep.cpp QueryPlan/LimitStep.cpp + QueryPlan/MaterializingStep.cpp QueryPlan/MergeSortingStep.cpp QueryPlan/MergingAggregatedStep.cpp + QueryPlan/MergingFinal.cpp QueryPlan/MergingSortedStep.cpp QueryPlan/OffsetStep.cpp QueryPlan/PartialSortingStep.cpp QueryPlan/QueryPlan.cpp QueryPlan/ReadFromPreparedSource.cpp - QueryPlan/ReadFromStorageStep.cpp QueryPlan/ReadNothingStep.cpp + QueryPlan/ReverseRowsStep.cpp QueryPlan/RollupStep.cpp + QueryPlan/SettingQuotaAndLimitsStep.cpp QueryPlan/TotalsHavingStep.cpp QueryPlan/UnionStep.cpp ResizeProcessor.cpp diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 5e03e1d178c..d7a1484d51d 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -5,39 +5,39 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include +#include #include #include #include -#include +#include #include -#include -#include +#include #include #include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include +#include +#include #if !defined(ARCADIA_BUILD) # include @@ -46,10 +46,8 @@ namespace DB { - namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int CANNOT_PARSE_TEXT; extern const int CANNOT_PARSE_ESCAPE_SEQUENCE; @@ -109,36 +107,25 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti { return HTTPResponse::HTTP_UNAUTHORIZED; } - else if (exception_code == ErrorCodes::CANNOT_PARSE_TEXT || - exception_code == ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE || - exception_code == ErrorCodes::CANNOT_PARSE_QUOTED_STRING || - exception_code == ErrorCodes::CANNOT_PARSE_DATE || - exception_code == ErrorCodes::CANNOT_PARSE_DATETIME || - exception_code == ErrorCodes::CANNOT_PARSE_NUMBER || - exception_code == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED || - exception_code == ErrorCodes::UNKNOWN_ELEMENT_IN_AST || - exception_code == ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE || - exception_code == ErrorCodes::TOO_DEEP_AST || - exception_code == ErrorCodes::TOO_BIG_AST || - exception_code == ErrorCodes::UNEXPECTED_AST_STRUCTURE || - exception_code == ErrorCodes::SYNTAX_ERROR || - exception_code == ErrorCodes::INCORRECT_DATA || - exception_code == ErrorCodes::TYPE_MISMATCH) + else if ( + exception_code == ErrorCodes::CANNOT_PARSE_TEXT || exception_code == ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE + || exception_code == ErrorCodes::CANNOT_PARSE_QUOTED_STRING || exception_code == ErrorCodes::CANNOT_PARSE_DATE + || exception_code == ErrorCodes::CANNOT_PARSE_DATETIME || exception_code == ErrorCodes::CANNOT_PARSE_NUMBER + || exception_code == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED || exception_code == ErrorCodes::UNKNOWN_ELEMENT_IN_AST + || exception_code == ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE || exception_code == ErrorCodes::TOO_DEEP_AST + || exception_code == ErrorCodes::TOO_BIG_AST || exception_code == ErrorCodes::UNEXPECTED_AST_STRUCTURE + || exception_code == ErrorCodes::SYNTAX_ERROR || exception_code == ErrorCodes::INCORRECT_DATA + || exception_code == ErrorCodes::TYPE_MISMATCH) { return HTTPResponse::HTTP_BAD_REQUEST; } - else if (exception_code == ErrorCodes::UNKNOWN_TABLE || - exception_code == ErrorCodes::UNKNOWN_FUNCTION || - exception_code == ErrorCodes::UNKNOWN_IDENTIFIER || - exception_code == ErrorCodes::UNKNOWN_TYPE || - exception_code == ErrorCodes::UNKNOWN_STORAGE || - exception_code == ErrorCodes::UNKNOWN_DATABASE || - exception_code == ErrorCodes::UNKNOWN_SETTING || - exception_code == ErrorCodes::UNKNOWN_DIRECTION_OF_SORTING || - exception_code == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION || - exception_code == ErrorCodes::UNKNOWN_FORMAT || - exception_code == ErrorCodes::UNKNOWN_DATABASE_ENGINE || - exception_code == ErrorCodes::UNKNOWN_TYPE_OF_QUERY) + else if ( + exception_code == ErrorCodes::UNKNOWN_TABLE || exception_code == ErrorCodes::UNKNOWN_FUNCTION + || exception_code == ErrorCodes::UNKNOWN_IDENTIFIER || exception_code == ErrorCodes::UNKNOWN_TYPE + || exception_code == ErrorCodes::UNKNOWN_STORAGE || exception_code == ErrorCodes::UNKNOWN_DATABASE + || exception_code == ErrorCodes::UNKNOWN_SETTING || exception_code == ErrorCodes::UNKNOWN_DIRECTION_OF_SORTING + || exception_code == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION || exception_code == ErrorCodes::UNKNOWN_FORMAT + || exception_code == ErrorCodes::UNKNOWN_DATABASE_ENGINE || exception_code == ErrorCodes::UNKNOWN_TYPE_OF_QUERY) { return HTTPResponse::HTTP_NOT_FOUND; } @@ -150,8 +137,7 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti { return HTTPResponse::HTTP_NOT_IMPLEMENTED; } - else if (exception_code == ErrorCodes::SOCKET_TIMEOUT || - exception_code == ErrorCodes::CANNOT_OPEN_FILE) + else if (exception_code == ErrorCodes::SOCKET_TIMEOUT || exception_code == ErrorCodes::CANNOT_OPEN_FILE) { return HTTPResponse::HTTP_SERVICE_UNAVAILABLE; } @@ -164,9 +150,7 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti } -static std::chrono::steady_clock::duration parseSessionTimeout( - const Poco::Util::AbstractConfiguration & config, - const HTMLForm & params) +static std::chrono::steady_clock::duration parseSessionTimeout(const Poco::Util::AbstractConfiguration & config, const HTMLForm & params) { unsigned session_timeout = config.getInt("default_session_timeout", 60); @@ -180,8 +164,9 @@ static std::chrono::steady_clock::duration parseSessionTimeout( throw Exception("Invalid session timeout: '" + session_timeout_str + "'", ErrorCodes::INVALID_SESSION_TIMEOUT); if (session_timeout > max_session_timeout) - throw Exception("Session timeout '" + session_timeout_str + "' is larger than max_session_timeout: " + toString(max_session_timeout) - + ". Maximum session timeout could be modified in configuration file.", + throw Exception( + "Session timeout '" + session_timeout_str + "' is larger than max_session_timeout: " + toString(max_session_timeout) + + ". Maximum session timeout could be modified in configuration file.", ErrorCodes::INVALID_SESSION_TIMEOUT); } @@ -209,8 +194,7 @@ void HTTPHandler::pushDelayedResults(Output & used_output) IReadableWriteBuffer * write_buf_concrete; ReadBufferPtr reread_buf; - if (write_buf - && (write_buf_concrete = dynamic_cast(write_buf.get())) + if (write_buf && (write_buf_concrete = dynamic_cast(write_buf.get())) && (reread_buf = write_buf_concrete->tryGetReadBuffer())) { read_buffers.emplace_back(reread_buf); @@ -223,9 +207,7 @@ void HTTPHandler::pushDelayedResults(Output & used_output) } -HTTPHandler::HTTPHandler(IServer & server_, const std::string & name) - : server(server_) - , log(&Poco::Logger::get(name)) +HTTPHandler::HTTPHandler(IServer & server_, const std::string & name) : server(server_), log(&Poco::Logger::get(name)) { server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } @@ -271,12 +253,12 @@ void HTTPHandler::processQuery( else { /// It is prohibited to mix different authorization schemes. - if (request.hasCredentials() - || params.has("user") - || params.has("password") - || params.has("quota_key")) + if (request.hasCredentials() || params.has("user") || params.has("password") || params.has("quota_key")) { - throw Exception("Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously", ErrorCodes::REQUIRED_PASSWORD); + throw Exception( + "Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods " + "simultaneously", + ErrorCodes::REQUIRED_PASSWORD); } } @@ -318,12 +300,13 @@ void HTTPHandler::processQuery( { std::string opentelemetry_traceparent = request.get("traceparent"); std::string error; - if (!context.getClientInfo().parseTraceparentHeader( - opentelemetry_traceparent, error)) + if (!context.getClientInfo().parseTraceparentHeader(opentelemetry_traceparent, error)) { - throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, + throw Exception( + ErrorCodes::BAD_REQUEST_PARAMETER, "Failed to parse OpenTelemetry traceparent header '{}': {}", - opentelemetry_traceparent, error); + opentelemetry_traceparent, + error); } context.getClientInfo().opentelemetry_tracestate = request.get("tracestate", ""); @@ -332,8 +315,7 @@ void HTTPHandler::processQuery( // Set the query id supplied by the user, if any, and also update the // OpenTelemetry fields. - context.setCurrentQueryId(params.get("query_id", - request.get("X-ClickHouse-Query-Id", ""))); + context.setCurrentQueryId(params.get("query_id", request.get("X-ClickHouse-Query-Id", ""))); /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). String http_response_compression_methods = request.get("Accept-Encoding", ""); @@ -351,6 +333,8 @@ void HTTPHandler::processQuery( http_response_compression_method = CompressionMethod::Gzip; else if (std::string::npos != http_response_compression_methods.find("deflate")) http_response_compression_method = CompressionMethod::Zlib; + else if (std::string::npos != http_response_compression_methods.find("xz")) + http_response_compression_method = CompressionMethod::Xz; } bool client_supports_http_compression = http_response_compression_method != CompressionMethod::None; @@ -360,8 +344,8 @@ void HTTPHandler::processQuery( bool internal_compression = params.getParsed("compress", false); /// At least, we should postpone sending of first buffer_size result bytes - size_t buffer_size_total = std::max( - params.getParsed("buffer_size", DBMS_DEFAULT_BUFFER_SIZE), static_cast(DBMS_DEFAULT_BUFFER_SIZE)); + size_t buffer_size_total + = std::max(params.getParsed("buffer_size", DBMS_DEFAULT_BUFFER_SIZE), static_cast(DBMS_DEFAULT_BUFFER_SIZE)); /// If it is specified, the whole result will be buffered. /// First ~buffer_size bytes will be buffered in memory, the remaining bytes will be stored in temporary file. @@ -393,23 +377,20 @@ void HTTPHandler::processQuery( const std::string tmp_path(context.getTemporaryVolume()->getDisk()->getPath()); const std::string tmp_path_template(tmp_path + "http_buffers/"); - auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &) - { - return WriteBufferFromTemporaryFile::create(tmp_path_template); - }; + auto create_tmp_disk_buffer + = [tmp_path_template](const WriteBufferPtr &) { return WriteBufferFromTemporaryFile::create(tmp_path_template); }; cascade_buffer2.emplace_back(std::move(create_tmp_disk_buffer)); } else { - auto push_memory_buffer_and_continue = [next_buffer = used_output.out_maybe_compressed] (const WriteBufferPtr & prev_buf) - { + auto push_memory_buffer_and_continue = [next_buffer = used_output.out_maybe_compressed](const WriteBufferPtr & prev_buf) { auto * prev_memory_buffer = typeid_cast(prev_buf.get()); if (!prev_memory_buffer) throw Exception("Expected MemoryWriteBuffer", ErrorCodes::LOGICAL_ERROR); auto rdbuf = prev_memory_buffer->tryGetReadBuffer(); - copyData(*rdbuf , *next_buffer); + copyData(*rdbuf, *next_buffer); return next_buffer; }; @@ -417,8 +398,8 @@ void HTTPHandler::processQuery( cascade_buffer2.emplace_back(push_memory_buffer_and_continue); } - used_output.out_maybe_delayed_and_compressed = std::make_shared( - std::move(cascade_buffer1), std::move(cascade_buffer2)); + used_output.out_maybe_delayed_and_compressed + = std::make_shared(std::move(cascade_buffer1), std::move(cascade_buffer2)); } else { @@ -444,13 +425,23 @@ void HTTPHandler::processQuery( std::unique_ptr in; - static const NameSet reserved_param_names{"compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", - "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check"}; + static const NameSet reserved_param_names{ + "compress", + "decompress", + "user", + "password", + "quota_key", + "query_id", + "stacktrace", + "buffer_size", + "wait_end_of_query", + "session_id", + "session_timeout", + "session_check"}; Names reserved_param_suffixes; - auto param_could_be_skipped = [&] (const String & name) - { + auto param_could_be_skipped = [&](const String & name) { /// Empty parameter appears when URL like ?&a=b or a=b&&c=d. Just skip them for user's convenience. if (name.empty()) return true; @@ -575,12 +566,10 @@ void HTTPHandler::processQuery( client_info.http_method = http_method; client_info.http_user_agent = request.get("User-Agent", ""); - auto append_callback = [&context] (ProgressCallback callback) - { + auto append_callback = [&context](ProgressCallback callback) { auto prev = context.getProgressCallback(); - context.setProgressCallback([prev, callback] (const Progress & progress) - { + context.setProgressCallback([prev, callback](const Progress & progress) { if (prev) prev(progress); @@ -590,14 +579,13 @@ void HTTPHandler::processQuery( /// While still no data has been sent, we will report about query execution progress by sending HTTP headers. if (settings.send_progress_in_http_headers) - append_callback([&used_output] (const Progress & progress) { used_output.out->onProgress(progress); }); + append_callback([&used_output](const Progress & progress) { used_output.out->onProgress(progress); }); if (settings.readonly > 0 && settings.cancel_http_readonly_queries_on_client_close) { Poco::Net::StreamSocket & socket = dynamic_cast(request).socket(); - append_callback([&context, &socket](const Progress &) - { + append_callback([&context, &socket](const Progress &) { /// Assume that at the point this method is called no one is reading data from the socket any more. /// True for read-only queries. try @@ -621,15 +609,17 @@ void HTTPHandler::processQuery( query_scope.emplace(context); - executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context, - [&response] (const String & current_query_id, const String & content_type, const String & format, const String & timezone) - { + executeQuery( + *in, + *used_output.out_maybe_delayed_and_compressed, + /* allow_into_outfile = */ false, + context, + [&response](const String & current_query_id, const String & content_type, const String & format, const String & timezone) { response.setContentType(content_type); response.add("X-ClickHouse-Query-Id", current_query_id); response.add("X-ClickHouse-Format", format); response.add("X-ClickHouse-Timezone", timezone); - } - ); + }); if (used_output.hasDelayed()) { @@ -642,8 +632,11 @@ void HTTPHandler::processQuery( used_output.out->finalize(); } -void HTTPHandler::trySendExceptionToClient(const std::string & s, int exception_code, - Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response, +void HTTPHandler::trySendExceptionToClient( + const std::string & s, + int exception_code, + Poco::Net::HTTPServerRequest & request, + Poco::Net::HTTPServerResponse & response, Output & used_output) { try @@ -652,17 +645,14 @@ void HTTPHandler::trySendExceptionToClient(const std::string & s, int exception_ /// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body /// to avoid reading part of the current request body in the next request. - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST - && response.getKeepAlive() - && !request.stream().eof() + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && response.getKeepAlive() && !request.stream().eof() && exception_code != ErrorCodes::HTTP_LENGTH_REQUIRED) { request.stream().ignore(std::numeric_limits::max()); } - bool auth_fail = exception_code == ErrorCodes::UNKNOWN_USER || - exception_code == ErrorCodes::WRONG_PASSWORD || - exception_code == ErrorCodes::REQUIRED_PASSWORD; + bool auth_fail = exception_code == ErrorCodes::UNKNOWN_USER || exception_code == ErrorCodes::WRONG_PASSWORD + || exception_code == ErrorCodes::REQUIRED_PASSWORD; if (auth_fail) { @@ -740,10 +730,12 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne with_stacktrace = params.getParsed("stacktrace", false); /// Workaround. Poco does not detect 411 Length Required case. - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding() && - !request.hasContentLength()) + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding() + && !request.hasContentLength()) { - throw Exception("The Transfer-Encoding is not chunked and there is no Content-Length header for POST request", ErrorCodes::HTTP_LENGTH_REQUIRED); + throw Exception( + "The Transfer-Encoding is not chunked and there is no Content-Length header for POST request", + ErrorCodes::HTTP_LENGTH_REQUIRED); } processQuery(context, request, params, response, used_output, query_scope); @@ -771,7 +763,7 @@ DynamicQueryHandler::DynamicQueryHandler(IServer & server_, const std::string & bool DynamicQueryHandler::customizeQueryParam(Context & context, const std::string & key, const std::string & value) { if (key == param_name) - return true; /// do nothing + return true; /// do nothing if (startsWith(key, "param_")) { @@ -786,7 +778,6 @@ bool DynamicQueryHandler::customizeQueryParam(Context & context, const std::stri std::string DynamicQueryHandler::getQuery(Poco::Net::HTTPServerRequest & request, HTMLForm & params, Context & context) { - if (likely(!startsWith(request.getContentType(), "multipart/form-data"))) { /// Part of the query can be passed in the 'query' parameter and the rest in the request body @@ -811,10 +802,16 @@ std::string DynamicQueryHandler::getQuery(Poco::Net::HTTPServerRequest & request } PredefinedQueryHandler::PredefinedQueryHandler( - IServer & server_, const NameSet & receive_params_, const std::string & predefined_query_ - , const CompiledRegexPtr & url_regex_, const std::unordered_map & header_name_with_regex_) - : HTTPHandler(server_, "PredefinedQueryHandler"), receive_params(receive_params_), predefined_query(predefined_query_) - , url_regex(url_regex_), header_name_with_capture_regex(header_name_with_regex_) + IServer & server_, + const NameSet & receive_params_, + const std::string & predefined_query_, + const CompiledRegexPtr & url_regex_, + const std::unordered_map & header_name_with_regex_) + : HTTPHandler(server_, "PredefinedQueryHandler") + , receive_params(receive_params_) + , predefined_query(predefined_query_) + , url_regex(url_regex_) + , header_name_with_capture_regex(header_name_with_regex_) { } @@ -834,8 +831,7 @@ void PredefinedQueryHandler::customizeContext(Poco::Net::HTTPServerRequest & req /// If in the configuration file, the handler's header is regex and contains named capture group /// We will extract regex named capture groups as query parameters - const auto & set_query_params = [&](const char * begin, const char * end, const CompiledRegexPtr & compiled_regex) - { + const auto & set_query_params = [&](const char * begin, const char * end, const CompiledRegexPtr & compiled_regex) { int num_captures = compiled_regex->NumberOfCapturingGroups() + 1; re2::StringPiece matches[num_captures]; @@ -880,16 +876,16 @@ std::string PredefinedQueryHandler::getQuery(Poco::Net::HTTPServerRequest & requ Poco::Net::HTTPRequestHandlerFactory * createDynamicHandlerFactory(IServer & server, const std::string & config_prefix) { std::string query_param_name = server.config().getString(config_prefix + ".handler.query_param_name", "query"); - return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory(server, std::move(query_param_name)), server.config(), config_prefix); + return addFiltersFromConfig( + new HandlingRuleHTTPHandlerFactory(server, std::move(query_param_name)), server.config(), config_prefix); } static inline bool capturingNamedQueryParam(NameSet receive_params, const CompiledRegexPtr & compiled_regex) { const auto & capturing_names = compiled_regex->NamedCapturingGroups(); - return std::count_if(capturing_names.begin(), capturing_names.end(), [&](const auto & iterator) - { - return std::count_if(receive_params.begin(), receive_params.end(), - [&](const auto & param_name) { return param_name == iterator.first; }); + return std::count_if(capturing_names.begin(), capturing_names.end(), [&](const auto & iterator) { + return std::count_if( + receive_params.begin(), receive_params.end(), [&](const auto & param_name) { return param_name == iterator.first; }); }); } @@ -898,8 +894,10 @@ static inline CompiledRegexPtr getCompiledRegex(const std::string & expression) auto compiled_regex = std::make_shared(expression); if (!compiled_regex->ok()) - throw Exception("Cannot compile re2: " + expression + " for http handling rule, error: " + - compiled_regex->error() + ". Look at https://github.com/google/re2/wiki/Syntax for reference.", ErrorCodes::CANNOT_COMPILE_REGEXP); + throw Exception( + "Cannot compile re2: " + expression + " for http handling rule, error: " + compiled_regex->error() + + ". Look at https://github.com/google/re2/wiki/Syntax for reference.", + ErrorCodes::CANNOT_COMPILE_REGEXP); return compiled_regex; } @@ -909,7 +907,8 @@ Poco::Net::HTTPRequestHandlerFactory * createPredefinedHandlerFactory(IServer & Poco::Util::AbstractConfiguration & configuration = server.config(); if (!configuration.has(config_prefix + ".handler.query")) - throw Exception("There is no path '" + config_prefix + ".handler.query" + "' in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG); + throw Exception( + "There is no path '" + config_prefix + ".handler.query" + "' in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG); std::string predefined_query = configuration.getString(config_prefix + ".handler.query"); NameSet analyze_receive_params = analyzeReceiveQueryParams(predefined_query); @@ -940,14 +939,22 @@ Poco::Net::HTTPRequestHandlerFactory * createPredefinedHandlerFactory(IServer & auto regex = getCompiledRegex(url_expression); if (capturingNamedQueryParam(analyze_receive_params, regex)) - return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory( - server, std::move(analyze_receive_params), std::move(predefined_query), std::move(regex), - std::move(headers_name_with_regex)), configuration, config_prefix); + return addFiltersFromConfig( + new HandlingRuleHTTPHandlerFactory( + server, + std::move(analyze_receive_params), + std::move(predefined_query), + std::move(regex), + std::move(headers_name_with_regex)), + configuration, + config_prefix); } - return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory( - server, std::move(analyze_receive_params), std::move(predefined_query), CompiledRegexPtr{} ,std::move(headers_name_with_regex)), - configuration, config_prefix); + return addFiltersFromConfig( + new HandlingRuleHTTPHandlerFactory( + server, std::move(analyze_receive_params), std::move(predefined_query), CompiledRegexPtr{}, std::move(headers_name_with_regex)), + configuration, + config_prefix); } } diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 3b04c6cd632..94f37254b91 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -7,11 +7,12 @@ #include #include #include -#include +#include #include #include #include #include +#include namespace DB @@ -94,24 +95,25 @@ Pipe IStorage::read( void IStorage::read( QueryPlan & query_plan, - TableLockHolder table_lock, - StorageMetadataPtr metadata_snapshot, - StreamLocalLimits & limits, - SizeLimits & leaf_limits, - std::shared_ptr quota, const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info, - std::shared_ptr context, + const Context & context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { - auto read_step = std::make_unique( - std::move(table_lock), std::move(metadata_snapshot), limits, leaf_limits, std::move(quota), shared_from_this(), - column_names, query_info, std::move(context), processed_stage, max_block_size, num_streams); - - read_step->setStepDescription("Read from " + getName()); - query_plan.addStep(std::move(read_step)); + auto pipe = read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + if (pipe.empty()) + { + auto header = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + InterpreterSelectQuery::addEmptySourceToQueryPlan(query_plan, header, query_info); + } + else + { + auto read_step = std::make_unique(std::move(pipe), getName()); + query_plan.addStep(std::move(read_step)); + } } Pipe IStorage::alterPartition( diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 493169e56a1..df26f1c54bd 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -48,6 +48,7 @@ using Processors = std::vector; class Pipe; class QueryPlan; +using QueryPlanPtr = std::unique_ptr; class StoragePolicy; using StoragePolicyPtr = std::shared_ptr; @@ -285,17 +286,13 @@ public: /// Default implementation creates ReadFromStorageStep and uses usual read. virtual void read( QueryPlan & query_plan, - TableLockHolder table_lock, - StorageMetadataPtr metadata_snapshot, - StreamLocalLimits & limits, - SizeLimits & leaf_limits, - std::shared_ptr quota, - const Names & column_names, - SelectQueryInfo & query_info, - std::shared_ptr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - unsigned num_streams); + const Names & /*column_names*/, + const StorageMetadataPtr & /*metadata_snapshot*/, + SelectQueryInfo & /*query_info*/, + const Context & /*context*/, + QueryProcessingStage::Enum /*processed_stage*/, + size_t /*max_block_size*/, + unsigned /*num_streams*/); /** Writes the data to a table. * Receives a description of the query, which can contain information about the data write method. diff --git a/src/Storages/MergeTree/BoolMask.h b/src/Storages/MergeTree/BoolMask.h index 3538c581137..c26a0ed6c58 100644 --- a/src/Storages/MergeTree/BoolMask.h +++ b/src/Storages/MergeTree/BoolMask.h @@ -9,17 +9,17 @@ struct BoolMask BoolMask() {} BoolMask(bool can_be_true_, bool can_be_false_) : can_be_true(can_be_true_), can_be_false(can_be_false_) {} - BoolMask operator &(const BoolMask & m) + BoolMask operator &(const BoolMask & m) const { - return BoolMask(can_be_true && m.can_be_true, can_be_false || m.can_be_false); + return {can_be_true && m.can_be_true, can_be_false || m.can_be_false}; } - BoolMask operator |(const BoolMask & m) + BoolMask operator |(const BoolMask & m) const { - return BoolMask(can_be_true || m.can_be_true, can_be_false && m.can_be_false); + return {can_be_true || m.can_be_true, can_be_false && m.can_be_false}; } - BoolMask operator !() + BoolMask operator !() const { - return BoolMask(can_be_false, can_be_true); + return {can_be_false, can_be_true}; } /// If mask is (true, true), then it can no longer change under operation |. diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp index 6b00215fd26..1f194092f5f 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB @@ -71,13 +72,13 @@ EphemeralLockInZooKeeper::~EphemeralLockInZooKeeper() EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( const String & block_numbers_path, const String & path_prefix, const String & temp_path, zkutil::ZooKeeper & zookeeper_) - : zookeeper(zookeeper_) + : zookeeper(&zookeeper_) { std::vector holders; while (true) { Coordination::Stat partitions_stat; - Strings partitions = zookeeper.getChildren(block_numbers_path, &partitions_stat); + Strings partitions = zookeeper->getChildren(block_numbers_path, &partitions_stat); if (holders.size() < partitions.size()) { @@ -85,7 +86,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( for (size_t i = 0; i < partitions.size() - holders.size(); ++i) { String path = temp_path + "/abandonable_lock-"; - holder_futures.push_back(zookeeper.asyncCreate(path, {}, zkutil::CreateMode::EphemeralSequential)); + holder_futures.push_back(zookeeper->asyncCreate(path, {}, zkutil::CreateMode::EphemeralSequential)); } for (auto & future : holder_futures) { @@ -104,7 +105,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( lock_ops.push_back(zkutil::makeCheckRequest(block_numbers_path, partitions_stat.version)); Coordination::Responses lock_responses; - Coordination::Error rc = zookeeper.tryMulti(lock_ops, lock_responses); + Coordination::Error rc = zookeeper->tryMulti(lock_ops, lock_responses); if (rc == Coordination::Error::ZBADVERSION) { LOG_TRACE(&Poco::Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); @@ -131,13 +132,16 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( void EphemeralLocksInAllPartitions::unlock() { + if (!zookeeper) + return; + std::vector futures; for (const auto & lock : locks) { Coordination::Requests unlock_ops; unlock_ops.emplace_back(zkutil::makeRemoveRequest(lock.path, -1)); unlock_ops.emplace_back(zkutil::makeRemoveRequest(lock.holder_path, -1)); - futures.push_back(zookeeper.asyncMulti(unlock_ops)); + futures.push_back(zookeeper->asyncMulti(unlock_ops)); } for (auto & future : futures) diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.h b/src/Storages/MergeTree/EphemeralLockInZooKeeper.h index b85761e0b15..007768aea3a 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.h +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.h @@ -1,9 +1,14 @@ #pragma once +#include "ReplicatedMergeTreeMutationEntry.h" + #include #include #include +#include +#include + namespace DB { @@ -87,13 +92,30 @@ private: /// Acquires block number locks in all partitions. -class EphemeralLocksInAllPartitions : private boost::noncopyable +class EphemeralLocksInAllPartitions : public boost::noncopyable { public: EphemeralLocksInAllPartitions( const String & block_numbers_path, const String & path_prefix, const String & temp_path, zkutil::ZooKeeper & zookeeper_); + EphemeralLocksInAllPartitions() = default; + + EphemeralLocksInAllPartitions(EphemeralLocksInAllPartitions && rhs) noexcept + : zookeeper(rhs.zookeeper) + , locks(std::move(rhs.locks)) + { + rhs.zookeeper = nullptr; + } + + EphemeralLocksInAllPartitions & operator=(EphemeralLocksInAllPartitions && rhs) noexcept + { + zookeeper = rhs.zookeeper; + rhs.zookeeper = nullptr; + locks = std::move(rhs.locks); + return *this; + } + struct LockInfo { String path; @@ -110,8 +132,51 @@ public: ~EphemeralLocksInAllPartitions(); private: - zkutil::ZooKeeper & zookeeper; + zkutil::ZooKeeper * zookeeper = nullptr; std::vector locks; }; + +/// This class allows scoped manipulations with block numbers locked in certain partitions +/// See StorageReplicatedMergeTree::allocateBlockNumbersInAffectedPartitions and alter()/mutate() methods +class PartitionBlockNumbersHolder +{ +public: + PartitionBlockNumbersHolder(const PartitionBlockNumbersHolder &) = delete; + PartitionBlockNumbersHolder & operator=(const PartitionBlockNumbersHolder &) = delete; + + using BlockNumbersType = ReplicatedMergeTreeMutationEntry::BlockNumbersType; + + PartitionBlockNumbersHolder() = default; + PartitionBlockNumbersHolder( + BlockNumbersType block_numbers_, std::optional locked_block_numbers_holder) + : block_numbers(std::move(block_numbers_)) + , multiple_partitions_holder(std::move(locked_block_numbers_holder)) + { + } + PartitionBlockNumbersHolder( + BlockNumbersType block_numbers_, std::optional locked_block_numbers_holder) + : block_numbers(std::move(block_numbers_)) + , single_partition_holder(std::move(locked_block_numbers_holder)) + { + } + + PartitionBlockNumbersHolder & operator=(PartitionBlockNumbersHolder &&) = default; + + const BlockNumbersType & getBlockNumbers() const { return block_numbers; } + + void reset() + { + multiple_partitions_holder.reset(); + single_partition_holder.reset(); + block_numbers.clear(); + } + +private: + BlockNumbersType block_numbers; + + std::optional multiple_partitions_holder; + std::optional single_partition_holder; +}; + } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index f2b7882cbc9..7f031771373 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -23,24 +23,20 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include namespace ProfileEvents @@ -83,17 +79,6 @@ static Block getBlockWithPartColumn(const MergeTreeData::DataPartsVector & parts return Block{ColumnWithTypeAndName(std::move(column), std::make_shared(), "_part")}; } -/// Check if ORDER BY clause of the query has some expression. -static bool sortingDescriptionHasExpressions(const SortDescription & sort_description, const StorageMetadataPtr & metadata_snapshot) -{ - auto all_columns = metadata_snapshot->getColumns(); - for (const auto & sort_column : sort_description) - { - if (!all_columns.has(sort_column.column_name)) - return true; - } - return false; -} size_t MergeTreeDataSelectExecutor::getApproximateTotalRowsToRead( const MergeTreeData::DataPartsVector & parts, @@ -144,7 +129,7 @@ static RelativeSize convertAbsoluteSampleSizeToRelative(const ASTPtr & node, siz } -Pipe MergeTreeDataSelectExecutor::read( +QueryPlanPtr MergeTreeDataSelectExecutor::read( const Names & column_names_to_return, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -159,7 +144,7 @@ Pipe MergeTreeDataSelectExecutor::read( max_block_numbers_to_read); } -Pipe MergeTreeDataSelectExecutor::readFromParts( +QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( MergeTreeData::DataPartsVector parts, const Names & column_names_to_return, const StorageMetadataPtr & metadata_snapshot, @@ -555,7 +540,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( if (no_data) { LOG_DEBUG(log, "Sampling yields no data."); - return {}; + return std::make_unique(); } LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); @@ -725,13 +710,13 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( LOG_DEBUG(log, "Selected {} parts by partition key, {} parts by primary key, {} marks by primary key, {} marks to read from {} ranges", parts.size(), parts_with_ranges.size(), sum_marks_pk.load(std::memory_order_relaxed), sum_marks, sum_ranges); if (parts_with_ranges.empty()) - return {}; + return std::make_unique(); ProfileEvents::increment(ProfileEvents::SelectedParts, parts_with_ranges.size()); ProfileEvents::increment(ProfileEvents::SelectedRanges, sum_ranges); ProfileEvents::increment(ProfileEvents::SelectedMarks, sum_marks); - Pipe res; + QueryPlanPtr plan; /// Projection, that needed to drop columns, which have appeared by execution /// of some extra expressions, and to allow execute the same expressions later. @@ -752,7 +737,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( std::sort(column_names_to_read.begin(), column_names_to_read.end()); column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); - res = spreadMarkRangesAmongStreamsFinal( + plan = spreadMarkRangesAmongStreamsFinal( std::move(parts_with_ranges), num_streams, column_names_to_read, @@ -774,7 +759,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical()); auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActions(false); - res = spreadMarkRangesAmongStreamsWithOrder( + plan = spreadMarkRangesAmongStreamsWithOrder( std::move(parts_with_ranges), num_streams, column_names_to_read, @@ -790,7 +775,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( } else { - res = spreadMarkRangesAmongStreams( + plan = spreadMarkRangesAmongStreams( std::move(parts_with_ranges), num_streams, column_names_to_read, @@ -803,42 +788,52 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( reader_settings); } + if (!plan) + return std::make_unique(); + if (use_sampling) { - res.addSimpleTransform([&filter_expression, &filter_function](const Block & header) - { - return std::make_shared( - header, filter_expression, filter_function->getColumnName(), false); - }); + auto sampling_step = std::make_unique( + plan->getCurrentDataStream(), + filter_expression, + filter_function->getColumnName(), + false); + + sampling_step->setStepDescription("Sampling"); + plan->addStep(std::move(sampling_step)); } if (result_projection) { - res.addSimpleTransform([&result_projection](const Block & header) - { - return std::make_shared(header, result_projection); - }); + auto projection_step = std::make_unique(plan->getCurrentDataStream(), result_projection); + projection_step->setStepDescription("Remove unused columns after reading from storage"); + plan->addStep(std::move(projection_step)); } /// By the way, if a distributed query or query to a Merge table is made, then the `_sample_factor` column can have different values. if (sample_factor_column_queried) { - res.addSimpleTransform([used_sample_factor](const Block & header) - { - return std::make_shared>( - header, std::make_shared(), used_sample_factor, "_sample_factor"); - }); + ColumnWithTypeAndName column; + column.name = "_sample_factor"; + column.type = std::make_shared(); + column.column = column.type->createColumnConst(0, Field(used_sample_factor)); + + auto adding_column = std::make_unique(plan->getCurrentDataStream(), std::move(column)); + adding_column->setStepDescription("Add _sample_factor column"); + plan->addStep(std::move(adding_column)); } if (query_info.prewhere_info && query_info.prewhere_info->remove_columns_actions) { - res.addSimpleTransform([&query_info](const Block & header) - { - return std::make_shared(header, query_info.prewhere_info->remove_columns_actions); - }); + auto expression_step = std::make_unique( + plan->getCurrentDataStream(), + query_info.prewhere_info->remove_columns_actions); + + expression_step->setStepDescription("Remove unused columns after PREWHERE"); + plan->addStep(std::move(expression_step)); } - return res; + return plan; } namespace @@ -863,8 +858,20 @@ size_t roundRowsOrBytesToMarks( } +static QueryPlanPtr createPlanFromPipe(Pipe pipe, const std::string & description = "") +{ + auto plan = std::make_unique(); -Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( + std::string storage_name = "MergeTree"; + if (!description.empty()) + storage_name += ' ' + description; + + auto step = std::make_unique(std::move(pipe), storage_name); + plan->addStep(std::move(step)); + return plan; +} + +QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -958,7 +965,7 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( res.emplace_back(std::move(source)); } - return Pipe::unitePipes(std::move(res)); + return createPlanFromPipe(Pipe::unitePipes(std::move(res))); } else { @@ -982,19 +989,18 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( if (pipe.numOutputPorts() > 1) pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts())); - return pipe; + return createPlanFromPipe(std::move(pipe)); } } -static ExpressionActionsPtr createProjection(const Pipe & pipe, const MergeTreeData & data) +static ExpressionActionsPtr createProjection(const Block & header, const MergeTreeData & data) { - const auto & header = pipe.getHeader(); auto projection = std::make_shared(header.getNamesAndTypesList(), data.global_context); projection->add(ExpressionAction::project(header.getNames())); return projection; } -Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( +QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -1095,7 +1101,8 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( const size_t min_marks_per_stream = (sum_marks - 1) / num_streams + 1; bool need_preliminary_merge = (parts.size() > settings.read_in_order_two_level_merge_threshold); - size_t max_output_ports = 0; + + std::vector plans; for (size_t i = 0; i < num_streams && !parts.empty(); ++i) { @@ -1195,60 +1202,64 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( } } - auto pipe = Pipe::unitePipes(std::move(pipes)); + auto plan = createPlanFromPipe(Pipe::unitePipes(std::move(pipes)), " with order"); if (input_order_info->direction != 1) { - pipe.addSimpleTransform([](const Block & header) - { - return std::make_shared(header); - }); + auto reverse_step = std::make_unique(plan->getCurrentDataStream()); + plan->addStep(std::move(reverse_step)); } - max_output_ports = std::max(pipe.numOutputPorts(), max_output_ports); - res.emplace_back(std::move(pipe)); + plans.emplace_back(std::move(plan)); } if (need_preliminary_merge) { - /// If ORDER BY clause of the query contains some expression, - /// then those new columns should be added for the merge step, - /// and this should be done always, if there is at least one pipe that - /// has multiple output ports. - bool sorting_key_has_expression = sortingDescriptionHasExpressions(input_order_info->order_key_prefix_descr, metadata_snapshot); - bool force_sorting_key_transform = res.size() > 1 && max_output_ports > 1 && sorting_key_has_expression; + SortDescription sort_description; + for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j) + sort_description.emplace_back(metadata_snapshot->getSortingKey().column_names[j], + input_order_info->direction, 1); - for (auto & pipe : res) + for (auto & plan : plans) { - SortDescription sort_description; + /// Drop temporary columns, added by 'sorting_key_prefix_expr' + out_projection = createProjection(plan->getCurrentDataStream().header, data); - if (pipe.numOutputPorts() > 1 || force_sorting_key_transform) - { - for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j) - sort_description.emplace_back(metadata_snapshot->getSortingKey().column_names[j], - input_order_info->direction, 1); + auto expression_step = std::make_unique( + plan->getCurrentDataStream(), + sorting_key_prefix_expr); - /// Drop temporary columns, added by 'sorting_key_prefix_expr' - out_projection = createProjection(pipe, data); - pipe.addSimpleTransform([sorting_key_prefix_expr](const Block & header) - { - return std::make_shared(header, sorting_key_prefix_expr); - }); - } + expression_step->setStepDescription("Calculate sorting key prefix"); + plan->addStep(std::move(expression_step)); - if (pipe.numOutputPorts() > 1) - { - pipe.addTransform(std::make_shared( - pipe.getHeader(), pipe.numOutputPorts(), sort_description, max_block_size)); - } + auto merging_sorted = std::make_unique( + plan->getCurrentDataStream(), + sort_description, + max_block_size); + + merging_sorted->setStepDescription("Merge sorting mark ranges"); + plan->addStep(std::move(merging_sorted)); } } - return Pipe::unitePipes(std::move(res)); + if (plans.size() == 1) + return std::move(plans.front()); + + DataStreams input_streams; + for (const auto & plan : plans) + input_streams.emplace_back(plan->getCurrentDataStream()); + + const auto & common_header = plans.front()->getCurrentDataStream().header; + auto union_step = std::make_unique(std::move(input_streams), common_header); + + auto plan = std::make_unique(); + plan->unitePlans(std::move(union_step), std::move(plans)); + + return plan; } -Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( +QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -1316,11 +1327,11 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( parts_to_merge_ranges.push_back(parts.end()); } - Pipes partition_pipes; + std::vector partition_plans; for (size_t range_index = 0; range_index < parts_to_merge_ranges.size() - 1; ++range_index) { - Pipe pipe; + QueryPlanPtr plan; { Pipes pipes; @@ -1346,12 +1357,17 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( pipes.emplace_back(std::move(source_processor)); } - pipe = Pipe::unitePipes(std::move(pipes)); - } + if (pipes.empty()) + continue; - /// Drop temporary columns, added by 'sorting_key_expr' - if (!out_projection) - out_projection = createProjection(pipe, data); + auto pipe = Pipe::unitePipes(std::move(pipes)); + + /// Drop temporary columns, added by 'sorting_key_expr' + if (!out_projection) + out_projection = createProjection(pipe.getHeader(), data); + + plan = createPlanFromPipe(std::move(pipe), "with final"); + } /// If do_not_merge_across_partitions_select_final is true and there is only one part in partition /// with level > 0 then we won't postprocess this part @@ -1359,14 +1375,16 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( std::distance(parts_to_merge_ranges[range_index], parts_to_merge_ranges[range_index + 1]) == 1 && parts_to_merge_ranges[range_index]->data_part->info.level > 0) { - partition_pipes.emplace_back(std::move(pipe)); + partition_plans.emplace_back(std::move(plan)); continue; } - pipe.addSimpleTransform([&metadata_snapshot](const Block & header) - { - return std::make_shared(header, metadata_snapshot->getSortingKey().expression); - }); + auto expression_step = std::make_unique( + plan->getCurrentDataStream(), + metadata_snapshot->getSortingKey().expression); + + expression_step->setStepDescription("Calculate sorting key expression"); + plan->addStep(std::move(expression_step)); Names sort_columns = metadata_snapshot->getSortingKeyColumns(); SortDescription sort_description; @@ -1375,111 +1393,40 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( Names partition_key_columns = metadata_snapshot->getPartitionKey().column_names; - Block header = pipe.getHeader(); + const auto & header = plan->getCurrentDataStream().header; for (size_t i = 0; i < sort_columns_size; ++i) sort_description.emplace_back(header.getPositionByName(sort_columns[i]), 1, 1); - auto get_merging_processor = [&]() -> MergingTransformPtr - { - switch (data.merging_params.mode) - { - case MergeTreeData::MergingParams::Ordinary: - { - return std::make_shared(header, pipe.numOutputPorts(), sort_description, max_block_size); - } + auto final_step = std::make_unique( + plan->getCurrentDataStream(), + std::min(num_streams, settings.max_final_threads), + sort_description, + data.merging_params, + partition_key_columns, + max_block_size); - case MergeTreeData::MergingParams::Collapsing: - return std::make_shared( - header, pipe.numOutputPorts(), sort_description, data.merging_params.sign_column, true, max_block_size); + final_step->setStepDescription("Merge rows for FINAL"); + plan->addStep(std::move(final_step)); - case MergeTreeData::MergingParams::Summing: - return std::make_shared( - header, - pipe.numOutputPorts(), - sort_description, - data.merging_params.columns_to_sum, - partition_key_columns, - max_block_size); - - case MergeTreeData::MergingParams::Aggregating: - return std::make_shared(header, pipe.numOutputPorts(), sort_description, max_block_size); - - case MergeTreeData::MergingParams::Replacing: - return std::make_shared( - header, pipe.numOutputPorts(), sort_description, data.merging_params.version_column, max_block_size); - - case MergeTreeData::MergingParams::VersionedCollapsing: - return std::make_shared( - header, pipe.numOutputPorts(), sort_description, data.merging_params.sign_column, max_block_size); - - case MergeTreeData::MergingParams::Graphite: - throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR); - } - - __builtin_unreachable(); - }; - - if (num_streams <= 1 || sort_description.empty()) - { - pipe.addTransform(get_merging_processor()); - partition_pipes.emplace_back(std::move(pipe)); - continue; - } - - ColumnNumbers key_columns; - key_columns.reserve(sort_description.size()); - - for (auto & desc : sort_description) - { - if (!desc.column_name.empty()) - key_columns.push_back(header.getPositionByName(desc.column_name)); - else - key_columns.emplace_back(desc.column_number); - } - - pipe.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, num_streams, key_columns); - }); - - pipe.transform([&](OutputPortRawPtrs ports) - { - Processors processors; - std::vector output_ports; - processors.reserve(ports.size() + num_streams); - output_ports.reserve(ports.size()); - - for (auto & port : ports) - { - auto copier = std::make_shared(header, num_streams); - connect(*port, copier->getInputPort()); - output_ports.emplace_back(copier->getOutputs().begin()); - processors.emplace_back(std::move(copier)); - } - - for (size_t i = 0; i < num_streams; ++i) - { - auto merge = get_merging_processor(); - merge->setSelectorPosition(i); - auto input = merge->getInputs().begin(); - - /// Connect i-th merge with i-th input port of every copier. - for (size_t j = 0; j < ports.size(); ++j) - { - connect(*output_ports[j], *input); - ++output_ports[j]; - ++input; - } - - processors.emplace_back(std::move(merge)); - } - - return processors; - }); - partition_pipes.emplace_back(std::move(pipe)); + partition_plans.emplace_back(std::move(plan)); } - return Pipe::unitePipes(std::move(partition_pipes)); + if (partition_plans.empty()) + return {}; + + if (partition_plans.size() == 1) + return std::move(partition_plans.front()); + + auto result_header = partition_plans.front()->getCurrentDataStream().header; + DataStreams input_streams; + for (const auto & partition_plan : partition_plans) + input_streams.push_back(partition_plan->getCurrentDataStream()); + + auto union_step = std::make_unique(std::move(input_streams), result_header); + union_step->setStepDescription("Unite sources after FINAL"); + QueryPlanPtr plan = std::make_unique(); + plan->unitePlans(std::move(union_step), std::move(partition_plans)); + return plan; } /// Calculates a set of mark ranges, that could possibly contain keys, required by condition. diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 5894d6e044b..7fd0e580b79 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -24,7 +24,7 @@ public: */ using PartitionIdToMaxBlock = std::unordered_map; - Pipe read( + QueryPlanPtr read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -33,7 +33,7 @@ public: unsigned num_streams, const PartitionIdToMaxBlock * max_block_numbers_to_read = nullptr) const; - Pipe readFromParts( + QueryPlanPtr readFromParts( MergeTreeData::DataPartsVector parts, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -48,7 +48,7 @@ private: Poco::Logger * log; - Pipe spreadMarkRangesAmongStreams( + QueryPlanPtr spreadMarkRangesAmongStreams( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -61,7 +61,7 @@ private: const MergeTreeReaderSettings & reader_settings) const; /// out_projection - save projection only with columns, requested to read - Pipe spreadMarkRangesAmongStreamsWithOrder( + QueryPlanPtr spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -75,7 +75,7 @@ private: const MergeTreeReaderSettings & reader_settings, ExpressionActionsPtr & out_projection) const; - Pipe spreadMarkRangesAmongStreamsFinal( + QueryPlanPtr spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, size_t num_streams, const Names & column_names, diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h index cfcc3dec6f2..6229b13c47f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h @@ -35,9 +35,10 @@ struct ReplicatedMergeTreeMutationEntry /// Replica which initiated mutation String source_replica; - /// Accured numbers of blocks + /// Acquired block numbers /// partition_id -> block_number - std::map block_numbers; + using BlockNumbersType = std::map; + BlockNumbersType block_numbers; /// Mutation commands which will give to MUTATE_PART entries MutationCommands commands; diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index aa24ddcf33c..98e1880de54 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -27,8 +29,11 @@ public: size_t max_block_size, unsigned num_streams) override { - return MergeTreeDataSelectExecutor(part->storage) - .readFromParts({part}, column_names, metadata_snapshot, query_info, context, max_block_size, num_streams); + QueryPlan query_plan = + std::move(*MergeTreeDataSelectExecutor(part->storage) + .readFromParts({part}, column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)); + + return query_plan.convertToPipe(); } @@ -45,6 +50,16 @@ public: return part->storage.getVirtuals(); } + String getPartitionId() const + { + return part->info.partition_id; + } + + String getPartitionIDFromQuery(const ASTPtr & ast, const Context & context) const + { + return part->storage.getPartitionIDFromQuery(ast, context); + } + protected: StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) : IStorage(getIDFromPart(part_)) diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index 53c9b50cb9d..bb22a1d0395 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -2,11 +2,13 @@ #include #include #include -#include #include #include #include +#include +#include #include +#include #include #include #include @@ -32,6 +34,7 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, res.ast = command->ptr(); res.type = DELETE; res.predicate = command->predicate; + res.partition = command->partition; return res; } else if (command->type == ASTAlterCommand::UPDATE) @@ -40,6 +43,7 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, res.ast = command->ptr(); res.type = UPDATE; res.predicate = command->predicate; + res.partition = command->partition; for (const ASTPtr & assignment_ast : command->update_assignments->children) { const auto & assignment = assignment_ast->as(); @@ -124,6 +128,7 @@ std::shared_ptr MutationCommands::ast() const return res; } + void MutationCommands::writeText(WriteBuffer & out) const { std::stringstream commands_ss; diff --git a/src/Storages/MutationCommands.h b/src/Storages/MutationCommands.h index 6e641e42cff..0f031eb56e6 100644 --- a/src/Storages/MutationCommands.h +++ b/src/Storages/MutationCommands.h @@ -43,8 +43,10 @@ struct MutationCommand /// Columns with corresponding actions std::unordered_map column_to_update_expression; - /// For MATERIALIZE INDEX + /// For MATERIALIZE INDEX. String index_name; + + /// For MATERIALIZE INDEX, UPDATE and DELETE. ASTPtr partition; /// For reads, drops and etc. diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 412dadab5bb..a2c2325bcc1 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -22,10 +22,13 @@ #include #include #include -#include +#include #include #include #include +#include +#include +#include namespace ProfileEvents @@ -147,6 +150,21 @@ QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage(const Context Pipe StorageBuffer::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return plan.convertToPipe(); +} + +void StorageBuffer::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info, @@ -155,8 +173,6 @@ Pipe StorageBuffer::read( size_t max_block_size, unsigned num_streams) { - Pipe pipe_from_dst; - if (destination_id) { auto destination = DatabaseCatalog::instance().getTable(destination_id, context); @@ -182,8 +198,8 @@ Pipe StorageBuffer::read( query_info.input_order_info = query_info.order_optimizer->getInputOrder(destination_metadata_snapshot); /// The destination table has the same structure of the requested columns and we can simply read blocks from there. - pipe_from_dst = destination->read( - column_names, destination_metadata_snapshot, query_info, + destination->read( + query_plan, column_names, destination_metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); } else @@ -217,29 +233,49 @@ Pipe StorageBuffer::read( } else { - pipe_from_dst = destination->read( - columns_intersection, destination_metadata_snapshot, query_info, - context, processed_stage, max_block_size, num_streams); + destination->read( + query_plan, columns_intersection, destination_metadata_snapshot, query_info, + context, processed_stage, max_block_size, num_streams); - if (!pipe_from_dst.empty()) + if (query_plan.isInitialized()) { - pipe_from_dst.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, header_after_adding_defaults, - metadata_snapshot->getColumns(), context); - }); - pipe_from_dst.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared( - stream_header, header, ConvertingTransform::MatchColumnsMode::Name); - }); + auto adding_missed = std::make_unique( + query_plan.getCurrentDataStream(), + header_after_adding_defaults, + metadata_snapshot->getColumns(), context); + + adding_missed->setStepDescription("Add columns missing in destination table"); + query_plan.addStep(std::move(adding_missed)); + + auto converting = std::make_unique( + query_plan.getCurrentDataStream(), + header); + + converting->setStepDescription("Convert destination table columns to Buffer table structure"); + query_plan.addStep(std::move(converting)); } } } - pipe_from_dst.addTableLock(destination_lock); - pipe_from_dst.addStorageHolder(destination); + if (query_plan.isInitialized()) + { + StreamLocalLimits limits; + SizeLimits leaf_limits; + + /// Add table lock for destination table. + auto adding_limits_and_quota = std::make_unique( + query_plan.getCurrentDataStream(), + destination, + std::move(destination_lock), + limits, + leaf_limits, + nullptr, + nullptr); + + adding_limits_and_quota->setStepDescription("Lock destination table for Buffer"); + query_plan.addStep(std::move(adding_limits_and_quota)); + } } Pipe pipe_from_buffers; @@ -252,49 +288,73 @@ Pipe StorageBuffer::read( pipe_from_buffers = Pipe::unitePipes(std::move(pipes_from_buffers)); } - /// Convert pipes from table to structure from buffer. - if (!pipe_from_buffers.empty() && !pipe_from_dst.empty() - && !blocksHaveEqualStructure(pipe_from_buffers.getHeader(), pipe_from_dst.getHeader())) - { - pipe_from_dst.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, - pipe_from_buffers.getHeader(), - ConvertingTransform::MatchColumnsMode::Name); - }); - } + if (pipe_from_buffers.empty()) + return; + + QueryPlan buffers_plan; /** If the sources from the table were processed before some non-initial stage of query execution, * then sources from the buffers must also be wrapped in the processing pipeline before the same stage. */ if (processed_stage > QueryProcessingStage::FetchColumns) - pipe_from_buffers = QueryPipeline::getPipe( - InterpreterSelectQuery(query_info.query, context, std::move(pipe_from_buffers), - SelectQueryOptions(processed_stage)).execute().pipeline); - - if (query_info.prewhere_info) { - pipe_from_buffers.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, query_info.prewhere_info->prewhere_actions, - query_info.prewhere_info->prewhere_column_name, query_info.prewhere_info->remove_prewhere_column); - }); - - if (query_info.prewhere_info->alias_actions) + auto interpreter = InterpreterSelectQuery( + query_info.query, context, std::move(pipe_from_buffers), + SelectQueryOptions(processed_stage)); + interpreter.buildQueryPlan(buffers_plan); + } + else + { + if (query_info.prewhere_info) { pipe_from_buffers.addSimpleTransform([&](const Block & header) { - return std::make_shared(header, query_info.prewhere_info->alias_actions); + return std::make_shared( + header, query_info.prewhere_info->prewhere_actions, + query_info.prewhere_info->prewhere_column_name, query_info.prewhere_info->remove_prewhere_column); }); + + if (query_info.prewhere_info->alias_actions) + { + pipe_from_buffers.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, query_info.prewhere_info->alias_actions); + }); + } } + + auto read_from_buffers = std::make_unique(std::move(pipe_from_buffers)); + read_from_buffers->setStepDescription("Read from buffers of Buffer table"); + buffers_plan.addStep(std::move(read_from_buffers)); } - Pipes pipes; - pipes.emplace_back(std::move(pipe_from_dst)); - pipes.emplace_back(std::move(pipe_from_buffers)); - return Pipe::unitePipes(std::move(pipes)); + if (!query_plan.isInitialized()) + { + query_plan = std::move(buffers_plan); + return; + } + + auto result_header = buffers_plan.getCurrentDataStream().header; + + /// Convert structure from table to structure from buffer. + if (!blocksHaveEqualStructure(query_plan.getCurrentDataStream().header, result_header)) + { + auto converting = std::make_unique(query_plan.getCurrentDataStream(), result_header); + query_plan.addStep(std::move(converting)); + } + + DataStreams input_streams; + input_streams.emplace_back(query_plan.getCurrentDataStream()); + input_streams.emplace_back(buffers_plan.getCurrentDataStream()); + + std::vector> plans; + plans.emplace_back(std::make_unique(std::move(query_plan))); + plans.emplace_back(std::make_unique(std::move(buffers_plan))); + query_plan = QueryPlan(); + + auto union_step = std::make_unique(std::move(input_streams), result_header); + union_step->setStepDescription("Unite sources from Buffer table"); + query_plan.unitePlans(std::move(union_step), std::move(plans)); } diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 455560e22da..3c8c4b8a040 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -65,6 +65,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + bool supportsParallelInsert() const override { return true; } BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 9316a5742c1..237cd5c4017 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -482,6 +482,21 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( } Pipe StorageDistributed::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return plan.convertToPipe(); +} + +void StorageDistributed::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info, @@ -508,7 +523,8 @@ Pipe StorageDistributed::read( : ClusterProxy::SelectStreamFactory( header, processed_stage, StorageID{remote_database, remote_table}, scalars, has_virtual_shard_num_column, context.getExternalTables()); - return ClusterProxy::executeQuery(select_stream_factory, log, modified_query_ast, context, query_info); + ClusterProxy::executeQuery(query_plan, select_stream_factory, log, + modified_query_ast, context, query_info); } diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index c4f56f1c2a1..790592033d8 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -62,6 +62,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t /*max_block_size*/, + unsigned /*num_streams*/) override; + bool supportsParallelInsert() const override { return true; } BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 057df65278e..fba4adbbf96 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -22,6 +22,7 @@ #include #include +#include namespace DB @@ -107,6 +108,21 @@ QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage(cons } Pipe StorageMaterializedView::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return plan.convertToPipe(); +} + +void StorageMaterializedView::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & /*metadata_snapshot*/, SelectQueryInfo & query_info, @@ -122,11 +138,26 @@ Pipe StorageMaterializedView::read( if (query_info.order_optimizer) query_info.input_order_info = query_info.order_optimizer->getInputOrder(metadata_snapshot); - Pipe pipe = storage->read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - pipe.addTableLock(lock); - pipe.addStorageHolder(storage); + storage->read(query_plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - return pipe; + if (query_plan.isInitialized()) + { + StreamLocalLimits limits; + SizeLimits leaf_limits; + + /// Add table lock for destination table. + auto adding_limits_and_quota = std::make_unique( + query_plan.getCurrentDataStream(), + storage, + std::move(lock), + limits, + leaf_limits, + nullptr, + nullptr); + + adding_limits_and_quota->setStepDescription("Lock destination table for Buffer"); + query_plan.addStep(std::move(adding_limits_and_quota)); + } } BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 9563e68b4fc..fd7ecea5526 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -81,6 +81,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + Strings getDataPaths() const override; private: diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 9b15ae00fc8..3e6f99878a6 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -332,10 +332,13 @@ Pipe StorageMerge::createSources( if (has_table_virtual_column) { - pipe.addSimpleTransform([name = table_name](const Block & stream_header) + ColumnWithTypeAndName column; + column.name = "_table"; + column.type = std::make_shared(); + column.column = column.type->createColumnConst(0, Field(table_name)); + pipe.addSimpleTransform([&](const Block & stream_header) { - return std::make_shared>( - stream_header, std::make_shared(), name, "_table"); + return std::make_shared(stream_header, column); }); } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index cc889d76c9c..925f510f63a 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -173,17 +173,32 @@ StorageMergeTree::~StorageMergeTree() shutdown(); } -Pipe StorageMergeTree::read( +void StorageMergeTree::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum /*processed_stage*/, + size_t max_block_size, + unsigned num_streams) +{ + if (auto plan = reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)) + query_plan = std::move(*plan); +} + +Pipe StorageMergeTree::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, const size_t max_block_size, const unsigned num_streams) { - return reader.read(column_names, metadata_snapshot, query_info, - context, max_block_size, num_streams); + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return plan.convertToPipe(); } std::optional StorageMergeTree::totalRows() const diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index d765b8bb92a..6cddfe77fee 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -46,6 +46,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & /*metadata_snapshot*/, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + std::optional totalRows() const override; std::optional totalRowsByPartitionPredicate(const SelectQueryInfo &, const Context &) const override; std::optional totalBytes() const override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ac55df5885c..e7264610e3e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3643,7 +3643,9 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg return max_added_blocks; } -Pipe StorageReplicatedMergeTree::read( + +void StorageReplicatedMergeTree::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info, @@ -3660,10 +3662,27 @@ Pipe StorageReplicatedMergeTree::read( if (context.getSettingsRef().select_sequential_consistency) { auto max_added_blocks = getMaxAddedBlocks(); - return reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams, &max_added_blocks); + if (auto plan = reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams, &max_added_blocks)) + query_plan = std::move(*plan); + return; } - return reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams); + if (auto plan = reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)) + query_plan = std::move(*plan); +} + +Pipe StorageReplicatedMergeTree::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return plan.convertToPipe(); } @@ -3915,6 +3934,60 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer } +std::set StorageReplicatedMergeTree::getPartitionIdsAffectedByCommands( + const MutationCommands & commands, const Context & query_context) const +{ + std::set affected_partition_ids; + + for (const auto & command : commands) + { + if (!command.partition) + { + affected_partition_ids.clear(); + break; + } + + affected_partition_ids.insert( + getPartitionIDFromQuery(command.partition, query_context) + ); + } + + return affected_partition_ids; +} + + +PartitionBlockNumbersHolder StorageReplicatedMergeTree::allocateBlockNumbersInAffectedPartitions( + const MutationCommands & commands, const Context & query_context, const zkutil::ZooKeeperPtr & zookeeper) const +{ + const std::set mutation_affected_partition_ids = getPartitionIdsAffectedByCommands(commands, query_context); + + if (mutation_affected_partition_ids.size() == 1) + { + const auto & affected_partition_id = *mutation_affected_partition_ids.cbegin(); + auto block_number_holder = allocateBlockNumber(affected_partition_id, zookeeper); + if (!block_number_holder.has_value()) + return {}; + auto block_number = block_number_holder->getNumber(); /// Avoid possible UB due to std::move + return {{{affected_partition_id, block_number}}, std::move(block_number_holder)}; + } + else + { + /// TODO: Implement optimal block number aqcuisition algorithm in multiple (but not all) partitions + EphemeralLocksInAllPartitions lock_holder( + zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + + PartitionBlockNumbersHolder::BlockNumbersType block_numbers; + for (const auto & lock : lock_holder.getLocks()) + { + if (mutation_affected_partition_ids.empty() || mutation_affected_partition_ids.count(lock.partition_id)) + block_numbers[lock.partition_id] = lock.number; + } + + return {std::move(block_numbers), std::move(lock_holder)}; + } +} + + void StorageReplicatedMergeTree::alter( const AlterCommands & commands, const Context & query_context, TableLockHolder & table_lock_holder) { @@ -3942,7 +4015,7 @@ void StorageReplicatedMergeTree::alter( return queryToString(query); }; - auto zookeeper = getZooKeeper(); + const auto zookeeper = getZooKeeper(); std::optional alter_entry; std::optional mutation_znode; @@ -3953,10 +4026,6 @@ void StorageReplicatedMergeTree::alter( alter_entry.emplace(); mutation_znode.reset(); - /// We can safely read structure, because we guarded with alter_intention_lock - if (is_readonly) - throw Exception("Can't ALTER readonly table", ErrorCodes::TABLE_IS_READ_ONLY); - auto current_metadata = getInMemoryMetadataPtr(); StorageInMemoryMetadata future_metadata = *current_metadata; @@ -4029,27 +4098,23 @@ void StorageReplicatedMergeTree::alter( ops.emplace_back(zkutil::makeCreateRequest( zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); - std::optional lock_holder; - - /// Now we will prepare mutations record. - /// This code pretty same with mutate() function but process results slightly differently. + PartitionBlockNumbersHolder partition_block_numbers_holder; if (alter_entry->have_mutation) { - String mutations_path = zookeeper_path + "/mutations"; + const String mutations_path(zookeeper_path + "/mutations"); ReplicatedMergeTreeMutationEntry mutation_entry; - mutation_entry.source_replica = replica_name; - mutation_entry.commands = maybe_mutation_commands; mutation_entry.alter_version = new_metadata_version; + mutation_entry.source_replica = replica_name; + mutation_entry.commands = std::move(maybe_mutation_commands); + Coordination::Stat mutations_stat; zookeeper->get(mutations_path, &mutations_stat); - lock_holder.emplace( - zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); - - for (const auto & lock : lock_holder->getLocks()) - mutation_entry.block_numbers[lock.partition_id] = lock.number; + partition_block_numbers_holder = + allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper); + mutation_entry.block_numbers = partition_block_numbers_holder.getBlockNumbers(); mutation_entry.create_time = time(nullptr); ops.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); @@ -4060,6 +4125,11 @@ void StorageReplicatedMergeTree::alter( Coordination::Responses results; Coordination::Error rc = zookeeper->tryMulti(ops, results); + /// For the sake of constitency with mechanics of concurrent background process of assigning parts merge tasks + /// this placeholder must be held up until the moment of committing into ZK of the mutation entry + /// See ReplicatedMergeTreeMergePredicate::canMergeTwoParts() method + partition_block_numbers_holder.reset(); + if (rc == Coordination::Error::ZOK) { if (alter_entry->have_mutation) @@ -4398,7 +4468,7 @@ void StorageReplicatedMergeTree::rename(const String & new_path_to_table_data, c } -bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) +bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) const { { std::lock_guard lock(existing_nodes_cache_mutex); @@ -4420,7 +4490,7 @@ bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) std::optional StorageReplicatedMergeTree::allocateBlockNumber( - const String & partition_id, zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_block_id_path) + const String & partition_id, const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_block_id_path) const { /// Lets check for duplicates in advance, to avoid superfluous block numbers allocation Coordination::Requests deduplication_check_ops; @@ -5063,44 +5133,46 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const /// After all needed parts are mutated (i.e. all active parts have the mutation version greater than /// the version of this mutation), the mutation is considered done and can be deleted. - ReplicatedMergeTreeMutationEntry entry; - entry.source_replica = replica_name; - entry.commands = commands; + ReplicatedMergeTreeMutationEntry mutation_entry; + mutation_entry.source_replica = replica_name; + mutation_entry.commands = commands; - String mutations_path = zookeeper_path + "/mutations"; + const String mutations_path = zookeeper_path + "/mutations"; + const auto zookeeper = getZooKeeper(); /// Update the mutations_path node when creating the mutation and check its version to ensure that /// nodes for mutations are created in the same order as the corresponding block numbers. /// Should work well if the number of concurrent mutation requests is small. while (true) { - auto zookeeper = getZooKeeper(); - Coordination::Stat mutations_stat; zookeeper->get(mutations_path, &mutations_stat); - EphemeralLocksInAllPartitions block_number_locks( - zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + PartitionBlockNumbersHolder partition_block_numbers_holder = + allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper); - for (const auto & lock : block_number_locks.getLocks()) - entry.block_numbers[lock.partition_id] = lock.number; - - entry.create_time = time(nullptr); + mutation_entry.block_numbers = partition_block_numbers_holder.getBlockNumbers(); + mutation_entry.create_time = time(nullptr); + /// The following version check guarantees the linearizability property for any pair of mutations: + /// mutation with higher sequence number is guaranteed to have higher block numbers in every partition + /// (and thus will be applied strictly according to sequence numbers of mutations) Coordination::Requests requests; requests.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); requests.emplace_back(zkutil::makeCreateRequest( - mutations_path + "/", entry.toString(), zkutil::CreateMode::PersistentSequential)); + mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); Coordination::Responses responses; Coordination::Error rc = zookeeper->tryMulti(requests, responses); + partition_block_numbers_holder.reset(); + if (rc == Coordination::Error::ZOK) { const String & path_created = dynamic_cast(responses[1].get())->path_created; - entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); - LOG_TRACE(log, "Created mutation with ID {}", entry.znode_name); + mutation_entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); + LOG_TRACE(log, "Created mutation with ID {}", mutation_entry.znode_name); break; } else if (rc == Coordination::Error::ZBADVERSION) @@ -5112,7 +5184,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const throw Coordination::Exception("Unable to create a mutation znode", rc); } - waitMutation(entry.znode_name, query_context.getSettingsRef().mutations_sync); + waitMutation(mutation_entry.znode_name, query_context.getSettingsRef().mutations_sync); } void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t mutations_sync) const diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 5944c9ce3a8..e0eaacf5e71 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -96,6 +96,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & /*metadata_snapshot*/, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + std::optional totalRows() const override; std::optional totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const override; std::optional totalBytes() const override; @@ -506,8 +516,8 @@ private: /// Creates new block number if block with such block_id does not exist std::optional allocateBlockNumber( - const String & partition_id, zkutil::ZooKeeperPtr & zookeeper, - const String & zookeeper_block_id_path = ""); + const String & partition_id, const zkutil::ZooKeeperPtr & zookeeper, + const String & zookeeper_block_id_path = "") const; /** Wait until all replicas, including this, execute the specified action from the log. * If replicas are added at the same time, it can not wait the added replica . @@ -531,9 +541,9 @@ private: bool getFakePartCoveringAllPartsInPartition(const String & partition_id, MergeTreePartInfo & part_info, bool for_replace_partition = false); /// Check for a node in ZK. If it is, remember this information, and then immediately answer true. - std::unordered_set existing_nodes_cache; - std::mutex existing_nodes_cache_mutex; - bool existsNodeCached(const std::string & path); + mutable std::unordered_set existing_nodes_cache; + mutable std::mutex existing_nodes_cache_mutex; + bool existsNodeCached(const std::string & path) const; void getClearBlocksInPartitionOps(Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num); /// Remove block IDs from `blocks/` in ZooKeeper for the given partition ID in the given block number range. @@ -565,6 +575,11 @@ private: MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const override; void startBackgroundMovesIfNeeded() override; + + std::set getPartitionIdsAffectedByCommands(const MutationCommands & commands, const Context & query_context) const; + PartitionBlockNumbersHolder allocateBlockNumbersInAffectedPartitions( + const MutationCommands & commands, const Context & query_context, const zkutil::ZooKeeperPtr & zookeeper) const; + protected: /** If not 'attach', either creates a new table in ZK, or adds a replica to an existing table. */ diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index f710a1289aa..66f06264b82 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -16,6 +16,9 @@ #include #include #include +#include +#include +#include namespace DB { @@ -52,12 +55,25 @@ Pipe StorageView::read( const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum /*processed_stage*/, - const size_t /*max_block_size*/, - const unsigned /*num_streams*/) + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) { - Pipes pipes; + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return plan.convertToPipe(); +} +void StorageView::read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum /*processed_stage*/, + const size_t /*max_block_size*/, + const unsigned /*num_streams*/) +{ ASTPtr current_inner_query = metadata_snapshot->getSelectQuery().inner_query; if (query_info.view_query) @@ -68,25 +84,19 @@ Pipe StorageView::read( } InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names); - - auto pipeline = interpreter.execute().pipeline; + interpreter.buildQueryPlan(query_plan); /// It's expected that the columns read from storage are not constant. /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. - pipeline.addSimpleTransform([](const Block & header) - { - return std::make_shared(header); - }); + auto materializing = std::make_unique(query_plan.getCurrentDataStream()); + materializing->setStepDescription("Materialize constants after VIEW subquery"); + query_plan.addStep(std::move(materializing)); /// And also convert to expected structure. - pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, metadata_snapshot->getSampleBlockForColumns( - column_names, getVirtuals(), getStorageID()), ConvertingTransform::MatchColumnsMode::Name); - }); - - return QueryPipeline::getPipe(std::move(pipeline)); + auto header = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + auto converting = std::make_unique(query_plan.getCurrentDataStream(), header); + converting->setStepDescription("Convert VIEW subquery result to VIEW table structure"); + query_plan.addStep(std::move(converting)); } static ASTTableExpression * getFirstTableExpression(ASTSelectQuery & select_query) diff --git a/src/Storages/StorageView.h b/src/Storages/StorageView.h index 56c9b0b4b1f..6f894ce2775 100644 --- a/src/Storages/StorageView.h +++ b/src/Storages/StorageView.h @@ -30,6 +30,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + void replaceWithSubquery(ASTSelectQuery & select_query, ASTPtr & view_name, const StorageMetadataPtr & metadata_snapshot) const { replaceWithSubquery(select_query, metadata_snapshot->getSelectQuery().inner_query->clone(), view_name); diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/__init__.py b/tests/integration/test_mutations_in_partitions_of_merge_tree/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/cluster.xml b/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/cluster.xml new file mode 100644 index 00000000000..ec7c9b8e4f8 --- /dev/null +++ b/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/cluster.xml @@ -0,0 +1,16 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + + + \ No newline at end of file diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/logs_config.xml b/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/logs_config.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/logs_config.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py b/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py new file mode 100644 index 00000000000..c7858c2f74d --- /dev/null +++ b/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py @@ -0,0 +1,98 @@ +import pytest +import helpers.client +import helpers.cluster + + +cluster = helpers.cluster.ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/logs_config.xml', 'configs/cluster.xml'], + with_zookeeper=True, stay_alive=True) + +node2 = cluster.add_instance('node2', main_configs=['configs/logs_config.xml', 'configs/cluster.xml'], + with_zookeeper=True, stay_alive=True) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_trivial_alter_in_partition_merge_tree_without_where(started_cluster): + try: + name = "test_trivial_alter_in_partition_merge_tree_without_where" + node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query("CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format(name)) + node1.query("INSERT INTO {} VALUES (1, 2), (2, 3)".format(name)) + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} DELETE IN PARTITION 1 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +def test_trivial_alter_in_partition_merge_tree_with_where(started_cluster): + try: + name = "test_trivial_alter_in_partition_merge_tree_with_where" + node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query("CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format(name)) + node1.query("INSERT INTO {} VALUES (1, 2), (2, 3)".format(name)) + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + node1.query("ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] + node1.query("ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +def test_trivial_alter_in_partition_replicated_merge_tree(started_cluster): + try: + name = "test_trivial_alter_in_partition_replicated_merge_tree" + + node1.query("DROP TABLE IF EXISTS {}".format(name)) + node2.query("DROP TABLE IF EXISTS {}".format(name)) + + for node in (node1, node2): + node.query( + "CREATE TABLE {name} (p Int64, x Int64) ENGINE=ReplicatedMergeTree('/clickhouse/{name}', '{{instance}}') ORDER BY tuple() PARTITION BY p" + .format(name=name)) + + node1.query("INSERT INTO {} VALUES (1, 2)".format(name)) + node2.query("INSERT INTO {} VALUES (2, 3)".format(name)) + + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE 1 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + node1.query("ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] + node1.query("ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + node2.query("DROP TABLE IF EXISTS {}".format(name)) diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 0dcd9335a52..6ae63b54ec6 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -1,4 +1,8 @@ - select number from (select number from numbers(10000000) order by -number) limit 10 - select number from (select number from numbers_mt(100000000) order by -number) limit 10 + CREATE VIEW numbers_view AS SELECT number from numbers_mt(100000000) order by number desc + + select number from (select number from numbers(1500000000) order by -number) limit 10 + select number from (select number from numbers_mt(1500000000) order by -number) limit 10 + + select number from numbers_view limit 100 diff --git a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql index e7174c5b56b..9912e083777 100644 --- a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql +++ b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql @@ -10,9 +10,9 @@ SET max_threads=1; SET optimize_move_functions_out_of_any=0; SELECT 'LIMIT'; -SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one) LIMIT 1 SETTINGS distributed_group_by_no_merge=2; +SELECT * FROM (SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one)) ORDER BY shard_num LIMIT 1 SETTINGS distributed_group_by_no_merge=2; SELECT 'OFFSET'; -SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one) LIMIT 1, 1 SETTINGS distributed_group_by_no_merge=2; +SELECT * FROM (SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one)) ORDER BY shard_num LIMIT 1, 1 SETTINGS distributed_group_by_no_merge=2; SELECT 'ALIAS'; SELECT dummy AS d FROM remote('127.0.0.{2,3}', system.one) ORDER BY d SETTINGS distributed_group_by_no_merge=2; diff --git a/tests/queries/0_stateless/00302_http_compression.reference b/tests/queries/0_stateless/00302_http_compression.reference index b2ab0bdb76c..f3d28359515 100644 --- a/tests/queries/0_stateless/00302_http_compression.reference +++ b/tests/queries/0_stateless/00302_http_compression.reference @@ -58,13 +58,26 @@ 7 8 9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 < Content-Encoding: gzip < Content-Encoding: deflate < Content-Encoding: gzip < Content-Encoding: br +< Content-Encoding: xz 1 1 1 +1 +Hello, world Hello, world Hello, world Hello, world diff --git a/tests/queries/0_stateless/00302_http_compression.sh b/tests/queries/0_stateless/00302_http_compression.sh index f3e10028dfb..846a23a54c9 100755 --- a/tests/queries/0_stateless/00302_http_compression.sh +++ b/tests/queries/0_stateless/00302_http_compression.sh @@ -9,6 +9,7 @@ ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept- ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: gzip, deflate' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d; ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: zip, eflate' -d 'SELECT number FROM system.numbers LIMIT 10'; ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: br' -d 'SELECT number FROM system.numbers LIMIT 10' | brotli -d; +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: xz' -d 'SELECT number FROM system.numbers LIMIT 10' | xz -d; ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&enable_http_compression=1" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep --text '< Content-Encoding'; ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep --text '< Content-Encoding'; @@ -16,14 +17,17 @@ ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: gzip, deflate' -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep --text '< Content-Encoding'; ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: zip, eflate' -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep --text '< Content-Encoding'; ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: br' -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep --text '< Content-Encoding'; +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: xz' -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep --text '< Content-Encoding'; echo "SELECT 1" | ${CLICKHOUSE_CURL} -sS --data-binary @- "${CLICKHOUSE_URL}"; echo "SELECT 1" | gzip -c | ${CLICKHOUSE_CURL} -sS --data-binary @- -H 'Content-Encoding: gzip' "${CLICKHOUSE_URL}"; echo "SELECT 1" | brotli | ${CLICKHOUSE_CURL} -sS --data-binary @- -H 'Content-Encoding: br' "${CLICKHOUSE_URL}"; +echo "SELECT 1" | xz -c | ${CLICKHOUSE_CURL} -sS --data-binary @- -H 'Content-Encoding: xz' "${CLICKHOUSE_URL}"; echo "'Hello, world'" | ${CLICKHOUSE_CURL} -sS --data-binary @- "${CLICKHOUSE_URL}&query=SELECT"; echo "'Hello, world'" | gzip -c | ${CLICKHOUSE_CURL} -sS --data-binary @- -H 'Content-Encoding: gzip' "${CLICKHOUSE_URL}&query=SELECT"; echo "'Hello, world'" | brotli | ${CLICKHOUSE_CURL} -sS --data-binary @- -H 'Content-Encoding: br' "${CLICKHOUSE_URL}&query=SELECT"; +echo "'Hello, world'" | xz -c | ${CLICKHOUSE_CURL} -sS --data-binary @- -H 'Content-Encoding: xz' "${CLICKHOUSE_URL}&query=SELECT"; ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 0' | wc -c; diff --git a/tests/queries/0_stateless/01059_storage_file_brotli.reference b/tests/queries/0_stateless/01059_storage_file_brotli.reference index aae55b2873c..266e67c1445 100644 --- a/tests/queries/0_stateless/01059_storage_file_brotli.reference +++ b/tests/queries/0_stateless/01059_storage_file_brotli.reference @@ -4,4 +4,4 @@ 3000000 999999 1 255 1 255 -1 255 \ No newline at end of file +1 255 diff --git a/tests/queries/0_stateless/01508_explain_header.reference b/tests/queries/0_stateless/01508_explain_header.reference index 50216432e14..a1f0b44c224 100644 --- a/tests/queries/0_stateless/01508_explain_header.reference +++ b/tests/queries/0_stateless/01508_explain_header.reference @@ -3,5 +3,7 @@ Header: x UInt8 Expression (Before ORDER BY and SELECT) Header: _dummy UInt8 1 UInt8 - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) Header: dummy UInt8 + ReadFromStorage (SystemOne) + Header: dummy UInt8 diff --git a/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.reference b/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.reference new file mode 100644 index 00000000000..aad646c5a79 --- /dev/null +++ b/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.reference @@ -0,0 +1,80 @@ +DateTime +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 + +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 +2020-10-16 00:00:00 + + +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 + +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +DateTime64 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 + +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + + +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 + +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + diff --git a/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.sql b/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.sql new file mode 100644 index 00000000000..5a3aa23bcbf --- /dev/null +++ b/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.sql @@ -0,0 +1,70 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test(timestamp DateTime) ENGINE = MergeTree ORDER BY timestamp; + +INSERT INTO test VALUES ('2020-10-15 00:00:00'); +INSERT INTO test VALUES ('2020-10-15 12:00:00'); +INSERT INTO test VALUES ('2020-10-16 00:00:00'); + +SELECT 'DateTime'; +SELECT * FROM test WHERE timestamp != '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp == '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp > '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp >= '2020-10-15' ORDER by timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp < '2020-10-16' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp <= '2020-10-16' ORDER BY timestamp; +SELECT ''; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' != timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' == timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' < timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' <= timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-16' > timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-16' >= timestamp ORDER BY timestamp; +SELECT ''; + +DROP TABLE test; +CREATE TABLE test(timestamp DateTime64) ENGINE = MergeTree ORDER BY timestamp; + +INSERT INTO test VALUES ('2020-10-15 00:00:00'); +INSERT INTO test VALUES ('2020-10-15 12:00:00'); +INSERT INTO test VALUES ('2020-10-16 00:00:00'); + +SELECT 'DateTime64'; +SELECT * FROM test WHERE timestamp != '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp == '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp > '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp >= '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp < '2020-10-16' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp <= '2020-10-16' ORDER BY timestamp; +SELECT ''; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' != timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' == timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' < timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' <= timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-16' > timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-16' >= timestamp ORDER BY timestamp; +SELECT ''; + +DROP TABLE test; diff --git a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference index 709116b4746..422a076b0cb 100644 --- a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference +++ b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference @@ -3,7 +3,7 @@ 5 1 1 2 1 -3 4 +3 3 1 1 2 1 3 4 diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference index fc10b4707a9..adbb03ae018 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference @@ -8,4 +8,23 @@ ExpressionTransform AggregatingInOrderTransform × 3 (Expression) ExpressionTransform × 3 - (ReadFromStorage) + (SettingQuotaAndLimits) + (Expression) + ExpressionTransform × 3 + (Union) + (MergingSorted) + (Expression) + ExpressionTransform + (ReadFromStorage) + MergeTree 0 → 1 + (MergingSorted) + MergingSortedTransform 2 → 1 + (Expression) + ExpressionTransform × 2 + (ReadFromStorage) + MergeTree × 2 0 → 1 + (MergingSorted) + (Expression) + ExpressionTransform + (ReadFromStorage) + MergeTree 0 → 1 diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 4c4f7b29d66..6420eadfc09 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -159,9 +159,10 @@ 01533_collate_in_nullable 01542_collate_in_array 01543_collate_in_tuple +01545_url_file_format_settings 01546_log_queries_min_query_duration_ms 01547_query_log_current_database 01548_query_log_query_execution_ms 01552_dict_fixedstring 01555_system_distribution_queue_mask -01557_max_parallel_replicas_no_sample.sql +01557_max_parallel_replicas_no_sample.sql \ No newline at end of file diff --git a/utils/check-style/check-include b/utils/check-style/check-include index 35f94d6e706..b4f105ed0cb 100755 --- a/utils/check-style/check-include +++ b/utils/check-style/check-include @@ -19,6 +19,7 @@ inc="-I. \ -I./contrib/double-conversion \ -I./contrib/cityhash102/include \ -I./contrib/croaring \ +-I./contrib/miniselect/include \ -I./contrib/murmurhash/include \ -I./contrib/zookeeper/src/c/include \ -I./contrib/zookeeper/src/c/generated \ diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index a6ca642b985..0e55de496b7 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -10,6 +10,7 @@ v20.8.2.3-stable 2020-09-08 v20.7.4.11-stable 2020-10-09 v20.7.3.7-stable 2020-09-18 v20.7.2.30-stable 2020-08-31 +v20.6.9.1-stable 2020-11-10 v20.6.8.5-stable 2020-10-12 v20.6.7.4-stable 2020-09-18 v20.6.6.7-stable 2020-09-11