thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

added and successfully passed tests for content encoding and file() function, formatted code

Browse Source
This commit is contained in:
a.palagashvili 2020-11-11 04:50:56 +03:00
commit 124ef2f5fa
114 changed files with 2481 additions and 896 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.gitmodules vendored
View File

@ -17,7 +17,6 @@
[submodule "contrib/zlib-ng"]
path = contrib/zlib-ng
url = https://github.com/ClickHouse-Extras/zlib-ng.git
branch = clickhouse
[submodule "contrib/googletest"]
path = contrib/googletest
url = https://github.com/google/googletest.git
@ -191,9 +190,9 @@
path = contrib/croaring
url = https://github.com/RoaringBitmap/CRoaring
branch = v0.2.66
[submodule "contrib/miniselect"]
path = contrib/miniselect
url = https://github.com/danlark1/miniselect
[submodule "contrib/xz"]
path = contrib/xz
url = https://github.com/xz-mirror/xz
[submodule "contrib/fast-lzma2"]
path = contrib/fast-lzma2
url = https://github.com/conor42/fast-lzma2
path = contrib/xz
url = https://github.com/xz-mirror/xz

1
CMakeLists.txt
View File

@ -445,6 +445,7 @@ include (cmake/find/brotli.cmake)
include (cmake/find/protobuf.cmake)
include (cmake/find/grpc.cmake)
include (cmake/find/pdqsort.cmake)
include (cmake/find/miniselect.cmake)
include (cmake/find/hdfs3.cmake) # uses protobuf
include (cmake/find/poco.cmake)
include (cmake/find/curl.cmake)

2
cmake/find/miniselect.cmake Normal file
View File

@ -0,0 +1,2 @@
set(MINISELECT_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/miniselect/include)
message(STATUS "Using miniselect: ${MINISELECT_INCLUDE_DIR}")

1
contrib/miniselect vendored Submodule

@ -0,0 +1 @@
Subproject commit be0af6bd0b6eb044d1acc4f754b229972d99903a

2
docker/test/fasttest/run.sh
View File

@ -127,7 +127,7 @@ function clone_submodules
(
cd "$FASTTEST_SOURCE"
SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11 contrib/croaring contrib/xz)
SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11 contrib/croaring contrib/miniselect)
git submodule sync
git submodule update --init --recursive "${SUBMODULES_TO_UPDATE[@]}"

9
docker/test/stress/stress
View File

@ -17,13 +17,6 @@ def get_skip_list_cmd(path):
return ''
def run_perf_test(cmd, xmls_path, output_folder):
output_path = os.path.join(output_folder, "perf_stress_run.txt")
f = open(output_path, 'w')
p = Popen("{} --skip-tags=long --recursive --input-files {}".format(cmd, xmls_path), shell=True, stdout=f, stderr=f)
return p
def get_options(i):
options = ""
if 0 < i:
@ -75,8 +68,6 @@ if __name__ == "__main__":
args = parser.parse_args()
func_pipes = []
perf_process = None
perf_process = run_perf_test(args.perf_test_cmd, args.perf_test_xml_path, args.output_folder)
func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests, args.global_time_limit)
logging.info("Will wait functests to finish")

141
docs/en/development/adding_test_queries.md Normal file
View File

@ -0,0 +1,141 @@
# How to add test queries to ClickHouse CI
ClickHouse has hundreds (or even thousands) of features. Every commit get checked by a complex set of tests containing many thousands of test cases.
The core functionality is very well tested, but some corner-cases and different combinations of features can be uncovered with ClickHouse CI.
Most of the bugs/regressions we see happen in that 'grey area' where test coverage is poor.
And we are very interested in covering most of the possible scenarios and feature combinations used in real life by tests.
## Why adding tests
Why/when you should add a test case into ClickHouse code:
1) you use some complicated scenarios / feature combinations / you have some corner case which is probably not widely used
2) you see that certain behavior gets changed between version w/o notifications in the changelog
3) you just want to help to improve ClickHouse quality and ensure the features you use will not be broken in the future releases
4) once the test is added/accepted, you can be sure the corner case you check will never be accidentally broken.
5) you will be a part of great open-source community
6) your name will be visible in the `system.contributors` table!
7) you will make a world bit better :)
### Steps to do
#### Prerequisite
I assume you run some Linux machine (you can use docker / virtual machines on other OS) and any modern browser / internet connection, and you have some basic Linux & SQL skills.
Any highly specialized knowledge is not needed (so you don't need to know C++ or know something about how ClickHouse CI works).
#### Preparation
1) [create GitHub account](https://github.com/join) (if you haven't one yet)
2) [setup git](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/set-up-git)
```bash
# for Ubuntu
sudo apt-get update
sudo apt-get install git
git config --global user.name "John Doe" # fill with your name
git config --global user.email "email@example.com" # fill with your email
```
3) [fork ClickHouse project](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/fork-a-repo) - just open [https://github.com/ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse) and press fork button in the top right corner:
![fork repo](https://github-images.s3.amazonaws.com/help/bootcamp/Bootcamp-Fork.png)
4) clone your fork to some folder on your PC, for example, `~/workspace/ClickHouse`
```
mkdir ~/workspace && cd ~/workspace
git clone https://github.com/< your GitHub username>/ClickHouse
cd ClickHouse
git remote add upstream https://github.com/ClickHouse/ClickHouse
```
#### New branch for the test
1) create a new branch from the latest clickhouse master
```
cd ~/workspace/ClickHouse
git fetch upstream
git checkout -b name_for_a_branch_with_my_test upstream/master
```
#### Install & run clickhouse
1) install `clickhouse-server` (follow [official docs](https://clickhouse.tech/docs/en/getting-started/install/))
2) install test configurations (it will use Zookeeper mock implementation and adjust some settings)
```
cd ~/workspace/ClickHouse/tests/config
sudo ./install.sh
```
3) run clickhouse-server
```
sudo systemctl restart clickhouse-server
```
#### Creating the test file
1) find the number for your test - find the file with the biggest number in `tests/queries/0_stateless/`
```sh
$ cd ~/workspace/ClickHouse
$ ls tests/queries/0_stateless/[0-9]*.reference | tail -n 1
tests/queries/0_stateless/01520_client_print_query_id.reference
```
Currently, the last number for the test is `01520`, so my test will have the number `01521`
2) create an SQL file with the next number and name of the feature you test
```sh
touch tests/queries/0_stateless/01521_dummy_test.sql
```
3) edit SQL file with your favorite editor (see hint of creating tests below)
```sh
vim tests/queries/0_stateless/01521_dummy_test.sql
```
4) run the test, and put the result of that into the reference file:
```
clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee tests/queries/0_stateless/01521_dummy_test.reference
```
5) ensure everything is correct, if the test output is incorrect (due to some bug for example), adjust the reference file using text editor.
#### How create good test
- test should be
- minimal - create only tables related to tested functionality, remove unrelated columns and parts of query
- fast - should not take longer than few seconds (better subseconds)
- correct - fails then feature is not working
- deteministic
- isolated / stateless
- don't rely on some environment things
- don't rely on timing when possible
- try to cover corner cases (zeros / Nulls / empty sets / throwing exceptions)
- to test that query return errors, you can put special comment after the query: `-- { serverError 60 }` or `-- { clientError 20 }`
- don't switch databases (unless necessary)
- you can create several table replicas on the same node if needed
- you can use one of the test cluster definitions when needed (see system.clusters)
- use `number` / `numbers_mt` / `zeros` / `zeros_mt` and similar for queries / to initialize data when appliable
- clean up the created objects after test and before the test (DROP IF EXISTS) - in case of some dirty state
- prefer sync mode of operations (mutations, merges, etc.)
- use other SQL files in the `0_stateless` folder as an example
- ensure the feature / feature combination you want to tests is not covered yet with existsing tests
#### Commit / push / create PR.
1) commit & push your changes
```sh
cd ~/workspace/ClickHouse
git add tests/queries/0_stateless/01521_dummy_test.sql
git add tests/queries/0_stateless/01521_dummy_test.reference
git commit # use some nice commit message when possible
git push origin HEAD
```
2) use a link which was shown during the push, to create a PR into the main repo
3) adjust the PR title and contents, in `Changelog category (leave one)` keep
`Build/Testing/Packaging Improvement`, fill the rest of the fields if you want.

2
docs/en/operations/settings/settings.md
View File

@ -384,7 +384,7 @@ Possible values:
- `'basic'` — Use basic parser.
ClickHouse can parse only the basic `YYYY-MM-DD HH:MM:SS` format. For example, `'2019-08-20 10:18:56'`.
ClickHouse can parse only the basic `YYYY-MM-DD HH:MM:SS` or `YYYY-MM-DD` format. For example, `'2019-08-20 10:18:56'` or `2019-08-20`.
Default value: `'basic'`.

37
docs/en/sql-reference/data-types/date.md
View File

@ -3,10 +3,45 @@ toc_priority: 47
toc_title: Date
---
# Date {#date}
# Date {#data_type-date}
A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2106, but the final fully-supported year is 2105).
The date value is stored without the time zone.
## Examples {#examples}
**1.** Creating a table with a `DateTime`-type column and inserting data into it:
``` sql
CREATE TABLE dt
(
`timestamp` Date,
`event_id` UInt8
)
ENGINE = TinyLog;
```
``` sql
INSERT INTO dt Values (1546300800, 1), ('2019-01-01', 2);
```
``` sql
SELECT * FROM dt;
```
``` text
┌──timestamp─┬─event_id─┐
│ 2019-01-01 │ 1 │
│ 2019-01-01 │ 2 │
└────────────┴──────────┘
```
## See Also {#see-also}
- [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md)
- [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime)
- [`DateTime` data type](../../sql-reference/data-types/datetime.md)
[Original article](https://clickhouse.tech/docs/en/data_types/date/) <!--hide-->

8
docs/en/sql-reference/table-functions/null.md
View File

@ -5,7 +5,7 @@ toc_title: null function
# null {#null-function}
Accepts an inserted data of the specified structure and immediately drops it away. The function is used for convenience writing tests and demonstrations.
Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately droped right after the query execution. The function is used for the convenience of test writing and demonstrations.
**Syntax**
@ -19,7 +19,7 @@ null('structure')
**Returned value**
A table with the specified structure, which is dropped right after the query execution.
A temporary `Null`-engine table with the specified structure.
**Example**
@ -36,6 +36,8 @@ INSERT INTO t SELECT * FROM numbers_mt(1000000000);
DROP TABLE IF EXISTS t;
```
See also: format **Null**.
See also:
- [Null table engine](../../engines/table-engines/special/null.md)
[Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/null/) <!--hide-->

43
docs/ru/sql-reference/table-functions/null.md Normal file
View File

@ -0,0 +1,43 @@
---
toc_priority: 53
toc_title: null функция
---
# null {#null-function}
Создает временную таблицу указанной структуры с движком [Null](../../engines/table-engines/special/null.md). В соответствии со свойствами движка, данные в таблице игнорируются, а сама таблица удаляется сразу после выполнения запроса. Функция используется для удобства написания тестов и демонстрационных примеров.
**Синтаксис**
``` sql
null('structure')
```
**Параметр**
- `structure` — список колонок и их типов. [String](../../sql-reference/data-types/string.md).
**Возвращаемое значение**
Временная таблица указанной структуры с движком `Null`.
**Пример**
Один запрос с функцией `null`:
``` sql
INSERT INTO function null('x UInt64') SELECT * FROM numbers_mt(1000000000);
```
заменяет три запроса:
```sql
CREATE TABLE t (x UInt64) ENGINE = Null;
INSERT INTO t SELECT * FROM numbers_mt(1000000000);
DROP TABLE IF EXISTS t;
```
См. также:
- [Движок таблиц Null](../../engines/table-engines/special/null.md)
[Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/null/) <!--hide-->

13
src/AggregateFunctions/QuantileExact.h
View File

@ -8,6 +8,7 @@
#include <Common/NaNUtils.h>
#include <Common/PODArray.h>
#include <miniselect/floyd_rivest_select.h>
namespace DB
{
@ -87,7 +88,7 @@ struct QuantileExact : QuantileExactBase<Value, QuantileExact<Value>>
{
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
std::nth_element(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm.
miniselect::floyd_rivest_select(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm.
return array[n];
}
@ -107,7 +108,7 @@ struct QuantileExact : QuantileExactBase<Value, QuantileExact<Value>>
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
std::nth_element(array.begin() + prev_n, array.begin() + n, array.end());
miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n, array.end());
result[indices[i]] = array[n];
prev_n = n;
@ -144,7 +145,7 @@ struct QuantileExactExclusive : public QuantileExact<Value>
else if (n < 1)
return static_cast<Float64>(array[0]);
std::nth_element(array.begin(), array.begin() + n - 1, array.end());
miniselect::floyd_rivest_select(array.begin(), array.begin() + n - 1, array.end());
auto nth_element = std::min_element(array.begin() + n, array.end());
return static_cast<Float64>(array[n - 1]) + (h - n) * static_cast<Float64>(*nth_element - array[n - 1]);
@ -173,7 +174,7 @@ struct QuantileExactExclusive : public QuantileExact<Value>
result[indices[i]] = static_cast<Float64>(array[0]);
else
{
std::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end());
miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n - 1, array.end());
auto nth_element = std::min_element(array.begin() + n, array.end());
result[indices[i]] = static_cast<Float64>(array[n - 1]) + (h - n) * static_cast<Float64>(*nth_element - array[n - 1]);
@ -209,7 +210,7 @@ struct QuantileExactInclusive : public QuantileExact<Value>
else if (n < 1)
return static_cast<Float64>(array[0]);
std::nth_element(array.begin(), array.begin() + n - 1, array.end());
miniselect::floyd_rivest_select(array.begin(), array.begin() + n - 1, array.end());
auto nth_element = std::min_element(array.begin() + n, array.end());
return static_cast<Float64>(array[n - 1]) + (h - n) * static_cast<Float64>(*nth_element - array[n - 1]);
@ -236,7 +237,7 @@ struct QuantileExactInclusive : public QuantileExact<Value>
result[indices[i]] = static_cast<Float64>(array[0]);
else
{
std::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end());
miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n - 1, array.end());
auto nth_element = std::min_element(array.begin() + n, array.end());
result[indices[i]] = static_cast<Float64>(array[n - 1]) + (h - n) * static_cast<Float64>(*nth_element - array[n - 1]);

5
src/AggregateFunctions/QuantileTiming.h
View File

@ -7,6 +7,7 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <miniselect/floyd_rivest_select.h>
namespace DB
{
@ -179,7 +180,7 @@ namespace detail
/// Sorting an array will not be considered a violation of constancy.
auto & array = elems;
std::nth_element(array.begin(), array.begin() + n, array.end());
miniselect::floyd_rivest_select(array.begin(), array.begin() + n, array.end());
quantile = array[n];
}
@ -200,7 +201,7 @@ namespace detail
? level * elems.size()
: (elems.size() - 1);
std::nth_element(array.begin() + prev_n, array.begin() + n, array.end());
miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n, array.end());
result[level_index] = array[n];
prev_n = n;

8
src/CMakeLists.txt
View File

@ -321,6 +321,7 @@ target_include_directories(clickhouse_common_io PUBLIC ${CMAKE_CURRENT_BINARY_DI
dbms_target_include_directories(PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/Core/include)
dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR})
dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${MINISELECT_INCLUDE_DIR})
if (ZSTD_LIBRARY)
dbms_target_link_libraries(PRIVATE ${ZSTD_LIBRARY})
@ -336,13 +337,6 @@ if (LZMA_LIBRARY)
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${LZMA_INCLUDE_DIR})
endif()
set (FAST_LZMA_LIBRARY fast-lzma2)
set (FAST_LZMA_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/fast-lzma2/)
if (FAST_LZMA_LIBRARY)
target_link_libraries (clickhouse_common_io PUBLIC ${FAST_LZMA_LIBRARY})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${FAST_LZMA_INCLUDE_DIR})
endif()
if (USE_ICU)
dbms_target_link_libraries (PRIVATE ${ICU_LIBRARIES})
dbms_target_include_directories (SYSTEM PRIVATE ${ICU_INCLUDE_DIRS})

5
src/Columns/ColumnArray.cpp
View File

@ -20,6 +20,7 @@
#include <Common/WeakHash.h>
#include <Common/HashTable/Hash.h>
#include <miniselect/floyd_rivest_select.h>
namespace DB
{
@ -782,7 +783,7 @@ void ColumnArray::getPermutationImpl(size_t limit, Permutation & res, Comparator
auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; };
if (limit)
std::partial_sort(res.begin(), res.begin() + limit, res.end(), less);
miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less);
else
std::sort(res.begin(), res.end(), less);
}
@ -835,7 +836,7 @@ void ColumnArray::updatePermutationImpl(size_t limit, Permutation & res, EqualRa
/// Since then we are working inside the interval.
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less);
miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less);
auto new_first = first;
for (auto j = first + 1; j < limit; ++j)
{

9
src/Columns/ColumnDecimal.cpp
View File

@ -8,6 +8,7 @@
#include <common/unaligned.h>
#include <ext/scope_guard.h>
#include <miniselect/floyd_rivest_select.h>
#include <IO/WriteHelpers.h>
@ -162,10 +163,10 @@ void ColumnDecimal<T>::updatePermutation(bool reverse, size_t limit, int, IColum
{
const auto& [first, last] = equal_ranges[i];
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last,
std::sort(res.begin() + first, res.begin() + last,
[this](size_t a, size_t b) { return data[a] > data[b]; });
else
std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last,
std::sort(res.begin() + first, res.begin() + last,
[this](size_t a, size_t b) { return data[a] < data[b]; });
auto new_first = first;
@ -193,10 +194,10 @@ void ColumnDecimal<T>::updatePermutation(bool reverse, size_t limit, int, IColum
/// Since then we are working inside the interval.
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last,
miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last,
[this](size_t a, size_t b) { return data[a] > data[b]; });
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last,
miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last,
[this](size_t a, size_t b) { return data[a] < data[b]; });
auto new_first = first;

5
src/Columns/ColumnDecimal.h
View File

@ -7,6 +7,7 @@
#include <Columns/IColumnImpl.h>
#include <Columns/ColumnVectorHelper.h>
#include <Core/Field.h>
#include <miniselect/floyd_rivest_select.h>
namespace DB
@ -253,9 +254,9 @@ protected:
sort_end = res.begin() + limit;
if (reverse)
std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; });
miniselect::floyd_rivest_partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; });
else
std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; });
miniselect::floyd_rivest_partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; });
}
};

9
src/Columns/ColumnFixedString.cpp
View File

@ -10,6 +10,7 @@
#include <Common/HashTable/Hash.h>
#include <ext/scope_guard.h>
#include <miniselect/floyd_rivest_select.h>
#include <DataStreams/ColumnGathererStream.h>
@ -157,9 +158,9 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir
if (limit)
{
if (reverse)
std::partial_sort(res.begin(), res.begin() + limit, res.end(), less<false>(*this));
miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less<false>(*this));
else
std::partial_sort(res.begin(), res.begin() + limit, res.end(), less<true>(*this));
miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less<true>(*this));
}
else
{
@ -217,9 +218,9 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu
/// Since then we are working inside the interval.
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less<false>(*this));
miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less<false>(*this));
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less<true>(*this));
miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less<true>(*this));
auto new_first = first;
for (auto j = first + 1; j < limit; ++j)

3
src/Columns/ColumnLowCardinality.cpp
View File

@ -8,6 +8,7 @@
#include <Common/WeakHash.h>
#include <ext/scope_guard.h>
#include <miniselect/floyd_rivest_select.h>
namespace DB
{
@ -393,7 +394,7 @@ void ColumnLowCardinality::updatePermutationImpl(size_t limit, Permutation & res
/// Since then we are working inside the interval.
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less);
miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less);
auto new_first = first;
for (auto j = first + 1; j < limit; ++j)

5
src/Columns/ColumnString.cpp
View File

@ -10,6 +10,7 @@
#include <common/unaligned.h>
#include <ext/scope_guard.h>
#include <miniselect/floyd_rivest_select.h>
namespace DB
{
@ -313,7 +314,7 @@ void ColumnString::getPermutationImpl(size_t limit, Permutation & res, Comparato
auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; };
if (limit)
std::partial_sort(res.begin(), res.begin() + limit, res.end(), less);
miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less);
else
std::sort(res.begin(), res.end(), less);
}
@ -365,7 +366,7 @@ void ColumnString::updatePermutationImpl(size_t limit, Permutation & res, EqualR
/// Since then we are working inside the interval.
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less);
miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less);
size_t new_first = first;
for (size_t j = first + 1; j < limit; ++j)

4
src/Columns/ColumnTuple.cpp
View File

@ -9,7 +9,7 @@
#include <Common/assert_cast.h>
#include <Common/WeakHash.h>
#include <Core/Field.h>
#include <miniselect/floyd_rivest_select.h>
namespace DB
{
@ -352,7 +352,7 @@ void ColumnTuple::getPermutationImpl(size_t limit, Permutation & res, LessOperat
if (limit)
{
std::partial_sort(res.begin(), res.begin() + limit, res.end(), less);
miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less);
}
else
{

10
src/Columns/ColumnVector.cpp
View File

@ -17,7 +17,7 @@
#include <ext/bit_cast.h>
#include <ext/scope_guard.h>
#include <pdqsort.h>
#include <miniselect/floyd_rivest_select.h>
#ifdef __SSE2__
#include <emmintrin.h>
@ -156,9 +156,9 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
res[i] = i;
if (reverse)
std::partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint));
miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint));
else
std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint));
miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint));
}
else
{
@ -254,9 +254,9 @@ void ColumnVector<T>::updatePermutation(bool reverse, size_t limit, int nan_dire
/// Since then, we are working inside the interval.
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint));
miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint));
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint));
miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint));
size_t new_first = first;
for (size_t j = first + 1; j < limit; ++j)

1
src/Core/Settings.h
View File

@ -399,6 +399,7 @@ class IColumn;
\
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
\
M(UInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \
M(UInt64, multiple_joins_rewriter_version, 0, "Obsolete setting, does nothing. Will be removed after 2021-03-31", 0) \
M(Bool, experimental_use_processors, true, "Obsolete setting, does nothing. Will be removed after 2020-11-29.", 0) \
M(Bool, force_optimize_skip_unused_shards_no_nested, false, "Obsolete setting, does nothing. Will be removed after 2020-12-01. Use force_optimize_skip_unused_shards_nesting instead.", 0) \

41
src/IO/LZMADeflatingWriteBuffer.cpp
View File

@ -13,10 +13,10 @@ LZMADeflatingWriteBuffer::LZMADeflatingWriteBuffer(
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment), out(std::move(out_))
{
// FL2_createCStreamMt(number of threads, flag of two dictionaries usage)
lstr = FL2_createCStreamMt(2, 0);
/* size_t res = */ FL2_initCStream(lstr, compression_level);
// lstr = FL2_createCStreamMt(2, 0);
// size_t res = FL2_initCStream(lstr, compression_level);
/*lstr = LZMA_STREAM_INIT;
lstr = LZMA_STREAM_INIT;
lstr.allocator = nullptr;
lstr.next_in = nullptr;
lstr.avail_in = 0;
@ -26,10 +26,13 @@ LZMADeflatingWriteBuffer::LZMADeflatingWriteBuffer(
// options for further compression
lzma_options_lzma opt_lzma2;
if (lzma_lzma_preset(&opt_lzma2, compression_level))
throw Exception(
std::string("lzma preset failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED);
throw Exception(ErrorCodes::LZMA_STREAM_ENCODER_FAILED, "lzma preset failed: lzma version: {}", LZMA_VERSION_STRING);
// LZMA_FILTER_X86 -
// LZMA2 - codec for *.xz files compression; LZMA is not suitable for this purpose
// VLI - variable length integer (in *.xz most integers encoded as VLI)
// LZMA_VLI_UNKNOWN (UINT64_MAX) - VLI value to denote that the value is unknown
lzma_filter filters[] = {
{.id = LZMA_FILTER_X86, .options = nullptr},
{.id = LZMA_FILTER_LZMA2, .options = &opt_lzma2},
@ -39,9 +42,10 @@ LZMADeflatingWriteBuffer::LZMADeflatingWriteBuffer(
if (ret != LZMA_OK)
throw Exception(
std::string("lzma stream encoder init failed: ") + std::to_string(ret) + "; lzma version: " + LZMA_VERSION_STRING,
ErrorCodes::LZMA_STREAM_ENCODER_FAILED);
*/
ErrorCodes::LZMA_STREAM_ENCODER_FAILED,
"lzma stream encoder init failed: error code: {} lzma version: {}",
ret,
LZMA_VERSION_STRING);
}
LZMADeflatingWriteBuffer::~LZMADeflatingWriteBuffer()
@ -50,7 +54,7 @@ LZMADeflatingWriteBuffer::~LZMADeflatingWriteBuffer()
{
finish();
//lzma_end(&lstr);
lzma_end(&lstr);
}
catch (...)
{
@ -60,7 +64,6 @@ LZMADeflatingWriteBuffer::~LZMADeflatingWriteBuffer()
void LZMADeflatingWriteBuffer::nextImpl()
{
/*
if (!offset())
return;
@ -74,27 +77,25 @@ void LZMADeflatingWriteBuffer::nextImpl()
lstr.next_out = reinterpret_cast<unsigned char *>(out->position());
lstr.avail_out = out->buffer().end() - out->position();
lzma_ret ret = lzma_code(&lstr, action);
out->position() = out->buffer().end() - lstr.avail_out;
if (ret == LZMA_STREAM_END)
return;
if (ret != LZMA_OK)
throw Exception(
std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING,
ErrorCodes::LZMA_STREAM_ENCODER_FAILED);
ErrorCodes::LZMA_STREAM_ENCODER_FAILED,
"lzma stream encoding failed: error code: {}; lzma_version: {}",
ret,
LZMA_VERSION_STRING);
} while (lstr.avail_in > 0 || lstr.avail_out == 0);
*/
}
void LZMADeflatingWriteBuffer::finish()
{
/*
if (finished)
return;
@ -109,7 +110,6 @@ void LZMADeflatingWriteBuffer::finish()
lzma_ret ret = lzma_code(&lstr, LZMA_FINISH);
out->position() = out->buffer().end() - lstr.avail_out;
if (ret == LZMA_STREAM_END)
{
finished = true;
@ -118,10 +118,11 @@ void LZMADeflatingWriteBuffer::finish()
if (ret != LZMA_OK)
throw Exception(
std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING,
ErrorCodes::LZMA_STREAM_ENCODER_FAILED);
ErrorCodes::LZMA_STREAM_ENCODER_FAILED,
"lzma stream encoding failed: error code: {}; lzma version: {}",
ret,
LZMA_VERSION_STRING);
} while (lstr.avail_out == 0);
*/
}
}

4
src/IO/LZMADeflatingWriteBuffer.h
View File

@ -4,7 +4,6 @@
#include <IO/WriteBuffer.h>
#include <lzma.h>
#include <fast-lzma2.h>
namespace DB
{
@ -27,7 +26,8 @@ private:
void nextImpl() override;
std::unique_ptr<WriteBuffer> out;
FL2_CStream * lstr;
lzma_stream lstr;
// FL2_CStream * lstr;
bool finished = false;
};
}

34
src/IO/LZMAInflatingReadBuffer.cpp
View File

@ -9,10 +9,6 @@ namespace ErrorCodes
LZMAInflatingReadBuffer::LZMAInflatingReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment), in(std::move(in_)), eof(false)
{
// FL2_createDStreamMt(number of threads)
lstr = FL2_createDStreamMt(2);
/* size_t res = */ FL2_initDStream(lstr);
/*
lstr = LZMA_STREAM_INIT;
lstr.allocator = nullptr;
lstr.next_in = nullptr;
@ -21,40 +17,48 @@ LZMAInflatingReadBuffer::LZMAInflatingReadBuffer(std::unique_ptr<ReadBuffer> in_
lstr.avail_out = 0;
// 500 mb
uint64_t memlimit = 500 << 20;
uint64_t memlimit = 500ULL << 20;
lzma_ret ret = lzma_stream_decoder(&lstr, memlimit, LZMA_CONCATENATED);
// lzma does not provide api for converting error code to string unlike zlib
if (ret != LZMA_OK)
throw Exception(
std::string("lzma_stream_decoder initialization failed: error code: ") + std::to_string(ret)
+ "; lzma version: " + LZMA_VERSION_STRING,
ErrorCodes::LZMA_STREAM_DECODER_FAILED);
*/
ErrorCodes::LZMA_STREAM_DECODER_FAILED,
"lzma_stream_decoder initialization failed: error code: {}; lzma version: {}",
ret,
LZMA_VERSION_STRING);
}
LZMAInflatingReadBuffer::~LZMAInflatingReadBuffer()
{
//lzma_end(&lstr);
lzma_end(&lstr);
}
bool LZMAInflatingReadBuffer::nextImpl()
{
/*
if (eof)
return false;
lzma_action action = LZMA_RUN;
if (!lstr.avail_in)
{
in->nextIfAtEnd();
lstr.next_in = reinterpret_cast<unsigned char *>(in->position());
lstr.avail_in = in->buffer().end() - in->position();
}
if (in->eof())
{
action = LZMA_FINISH;
}
lstr.next_out = reinterpret_cast<unsigned char *>(internal_buffer.begin());
lstr.avail_out = internal_buffer.size();
// std::cout << lstr.avail_in << " " << lstr.avail_out << std::endl;
lzma_ret ret = lzma_code(&lstr, LZMA_RUN);
lzma_ret ret = lzma_code(&lstr, action);
// std::cout << ret << std::endl;
in->position() = in->buffer().end() - lstr.avail_in;
working_buffer.resize(internal_buffer.size() - lstr.avail_out);
@ -69,7 +73,7 @@ bool LZMAInflatingReadBuffer::nextImpl()
{
throw Exception(
ErrorCodes::LZMA_STREAM_DECODER_FAILED,
"lzma decoder finished, but stream is still alive: error code: {}; lzma version: {}",
"lzma decoder finished, but input stream has not exceeded: error code: {}; lzma version: {}",
ret,
LZMA_VERSION_STRING);
}
@ -83,7 +87,5 @@ bool LZMAInflatingReadBuffer::nextImpl()
LZMA_VERSION_STRING);
return true;
*/
return true;
}
}

4
src/IO/LZMAInflatingReadBuffer.h
View File

@ -4,7 +4,6 @@
#include <IO/ReadBuffer.h>
#include <lzma.h>
#include <fast-lzma2.h>
namespace DB
{
@ -27,7 +26,8 @@ private:
bool nextImpl() override;
std::unique_ptr<ReadBuffer> in;
FL2_DStream * lstr;
lzma_stream lstr;
// FL2_DStream * lstr;
bool eof;
};

28
src/IO/ReadHelpers.cpp
View File

@ -817,7 +817,11 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
/// YYYY-MM-DD hh:mm:ss
static constexpr auto date_time_broken_down_length = 19;
/// YYYY-MM-DD
static constexpr auto date_broken_down_length = 10;
/// unix timestamp max length
static constexpr auto unix_timestamp_max_length = 10;
char s[date_time_broken_down_length];
@ -831,12 +835,15 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
++buf.position();
}
/// 2015-01-01 01:02:03
/// 2015-01-01 01:02:03 or 2015-01-01
if (s_pos == s + 4 && !buf.eof() && (*buf.position() < '0' || *buf.position() > '9'))
{
const size_t remaining_size = date_time_broken_down_length - (s_pos - s);
size_t size = buf.read(s_pos, remaining_size);
if (remaining_size != size)
const auto already_read_length = s_pos - s;
const size_t remaining_date_time_size = date_time_broken_down_length - already_read_length;
const size_t remaining_date_size = date_broken_down_length - already_read_length;
size_t size = buf.read(s_pos, remaining_date_time_size);
if (size != remaining_date_time_size && size != remaining_date_size)
{
s_pos[size] = 0;
@ -850,9 +857,16 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
UInt8 month = (s[5] - '0') * 10 + (s[6] - '0');
UInt8 day = (s[8] - '0') * 10 + (s[9] - '0');
UInt8 hour = (s[11] - '0') * 10 + (s[12] - '0');
UInt8 minute = (s[14] - '0') * 10 + (s[15] - '0');
UInt8 second = (s[17] - '0') * 10 + (s[18] - '0');
UInt8 hour = 0;
UInt8 minute = 0;
UInt8 second = 0;
if (size == remaining_date_time_size)
{
hour = (s[11] - '0') * 10 + (s[12] - '0');
minute = (s[14] - '0') * 10 + (s[15] - '0');
second = (s[17] - '0') * 10 + (s[18] - '0');
}
if (unlikely(year == 0))
datetime = 0;

13
src/IO/ReadHelpers.h
View File

@ -700,7 +700,7 @@ UInt128 stringToUUID(const String & str);
template <typename ReturnType = void>
ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut);
/** In YYYY-MM-DD hh:mm:ss format, according to specified time zone.
/** In YYYY-MM-DD hh:mm:ss or YYYY-MM-DD format, according to specified time zone.
* As an exception, also supported parsing of unix timestamp in form of decimal number.
*/
template <typename ReturnType = void>
@ -709,12 +709,17 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
/** Read 10 characters, that could represent unix timestamp.
* Only unix timestamp of 5-10 characters is supported.
* Then look at 5th character. If it is a number - treat whole as unix timestamp.
* If it is not a number - then parse datetime in YYYY-MM-DD hh:mm:ss format.
* If it is not a number - then parse datetime in YYYY-MM-DD hh:mm:ss or YYYY-MM-DD format.
*/
/// Optimistic path, when whole value is in buffer.
const char * s = buf.position();
if (s + 19 <= buf.buffer().end())
/// YYYY-MM-DD hh:mm:ss
static constexpr auto DateTimeStringInputSize = 19;
bool optimistic_path_for_date_time_input = s + DateTimeStringInputSize <= buf.buffer().end();
if (optimistic_path_for_date_time_input)
{
if (s[4] < '0' || s[4] > '9')
{
@ -731,7 +736,7 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
else
datetime = date_lut.makeDateTime(year, month, day, hour, minute, second);
buf.position() += 19;
buf.position() += DateTimeStringInputSize;
return ReturnType(true);
}
else

7
src/Interpreters/ClusterProxy/IStreamFactory.h
View File

@ -16,6 +16,9 @@ struct SelectQueryInfo;
class Pipe;
using Pipes = std::vector<Pipe>;
class QueryPlan;
using QueryPlanPtr = std::unique_ptr<QueryPlan>;
namespace ClusterProxy
{
@ -31,7 +34,9 @@ public:
const String & query, const ASTPtr & query_ast,
const Context & context, const ThrottlerPtr & throttler,
const SelectQueryInfo & query_info,
Pipes & res) = 0;
std::vector<QueryPlanPtr> & res,
Pipes & remote_pipes,
Pipes & delayed_pipes) = 0;
};
}

53
src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
View File

@ -14,6 +14,8 @@
#include <Processors/Sources/RemoteSource.h>
#include <Processors/Sources/DelayedSource.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Processors/QueryPlan/ConvertingStep.h>
namespace ProfileEvents
{
@ -69,37 +71,27 @@ SelectStreamFactory::SelectStreamFactory(
namespace
{
auto createLocalPipe(
const ASTPtr & query_ast, const Block & header, const Context & context, QueryProcessingStage::Enum processed_stage)
std::unique_ptr<QueryPlan> createLocalPlan(
const ASTPtr & query_ast,
const Block & header,
const Context & context,
QueryProcessingStage::Enum processed_stage)
{
checkStackSize();
InterpreterSelectQuery interpreter(query_ast, context, SelectQueryOptions(processed_stage));
auto query_plan = std::make_unique<QueryPlan>();
InterpreterSelectQuery interpreter(query_ast, context, SelectQueryOptions(processed_stage));
interpreter.buildQueryPlan(*query_plan);
auto pipeline = std::move(*query_plan->buildQueryPipeline());
/// Avoid going it out-of-scope for EXPLAIN
pipeline.addQueryPlan(std::move(query_plan));
/// Convert header structure to expected.
/// Also we ignore constants from result and replace it with constants from header.
/// It is needed for functions like `now64()` or `randConstant()` because their values may be different.
auto converting = std::make_unique<ConvertingStep>(query_plan->getCurrentDataStream(), header, true);
converting->setStepDescription("Convert block structure for query from local replica");
query_plan->addStep(std::move(converting));
pipeline.addSimpleTransform([&](const Block & source_header)
{
return std::make_shared<ConvertingTransform>(
source_header, header, ConvertingTransform::MatchColumnsMode::Name, true);
});
/** Materialization is needed, since from remote servers the constants come materialized.
* If you do not do this, different types (Const and non-Const) columns will be produced in different threads,
* And this is not allowed, since all code is based on the assumption that in the block stream all types are the same.
*/
/* Now we don't need to materialize constants, because RemoteBlockInputStream will ignore constant and take it from header.
* So, streams from different threads will always have the same header.
*/
pipeline.setMaxThreads(1);
return QueryPipeline::getPipe(std::move(pipeline));
return query_plan;
}
String formattedAST(const ASTPtr & ast)
@ -119,7 +111,9 @@ void SelectStreamFactory::createForShard(
const String &, const ASTPtr & query_ast,
const Context & context, const ThrottlerPtr & throttler,
const SelectQueryInfo &,
Pipes & pipes)
std::vector<QueryPlanPtr> & plans,
Pipes & remote_pipes,
Pipes & delayed_pipes)
{
bool add_agg_info = processed_stage == QueryProcessingStage::WithMergeableState;
bool add_totals = false;
@ -136,7 +130,7 @@ void SelectStreamFactory::createForShard(
auto emplace_local_stream = [&]()
{
pipes.emplace_back(createLocalPipe(modified_query_ast, header, context, processed_stage));
plans.emplace_back(createLocalPlan(modified_query_ast, header, context, processed_stage));
};
String modified_query = formattedAST(modified_query_ast);
@ -149,7 +143,7 @@ void SelectStreamFactory::createForShard(
if (!table_func_ptr)
remote_query_executor->setMainTable(main_table);
pipes.emplace_back(createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes));
remote_pipes.emplace_back(createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes));
};
const auto & settings = context.getSettingsRef();
@ -275,7 +269,10 @@ void SelectStreamFactory::createForShard(
}
if (try_results.empty() || local_delay < max_remote_delay)
return createLocalPipe(modified_query_ast, header, context, stage);
{
auto plan = createLocalPlan(modified_query_ast, header, context, stage);
return QueryPipeline::getPipe(std::move(*plan->buildQueryPipeline()));
}
else
{
std::vector<IConnectionPool::Entry> connections;
@ -290,7 +287,7 @@ void SelectStreamFactory::createForShard(
}
};
pipes.emplace_back(createDelayedPipe(header, lazily_create_stream, add_totals, add_extremes));
delayed_pipes.emplace_back(createDelayedPipe(header, lazily_create_stream, add_totals, add_extremes));
}
else
emplace_remote_stream();

4
src/Interpreters/ClusterProxy/SelectStreamFactory.h
View File

@ -39,7 +39,9 @@ public:
const String & query, const ASTPtr & query_ast,
const Context & context, const ThrottlerPtr & throttler,
const SelectQueryInfo & query_info,
Pipes & pipes) override;
std::vector<QueryPlanPtr> & plans,
Pipes & remote_pipes,
Pipes & delayed_pipes) override;
private:
const Block header;

49
src/Interpreters/ClusterProxy/executeQuery.cpp
View File

@ -7,6 +7,9 @@
#include <Interpreters/ProcessList.h>
#include <Parsers/queryToString.h>
#include <Processors/Pipe.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Processors/QueryPlan/ReadFromPreparedSource.h>
#include <Processors/QueryPlan/UnionStep.h>
#include <Storages/SelectQueryInfo.h>
@ -81,15 +84,19 @@ Context updateSettingsForCluster(const Cluster & cluster, const Context & contex
return new_context;
}
Pipe executeQuery(
void executeQuery(
QueryPlan & query_plan,
IStreamFactory & stream_factory, Poco::Logger * log,
const ASTPtr & query_ast, const Context & context, const SelectQueryInfo & query_info)
{
assert(log);
Pipes res;
const Settings & settings = context.getSettingsRef();
std::vector<QueryPlanPtr> plans;
Pipes remote_pipes;
Pipes delayed_pipes;
const std::string query = queryToString(query_ast);
Context new_context = updateSettingsForCluster(*query_info.cluster, context, settings, log);
@ -112,9 +119,43 @@ Pipe executeQuery(
throttler = user_level_throttler;
for (const auto & shard_info : query_info.cluster->getShardsInfo())
stream_factory.createForShard(shard_info, query, query_ast, new_context, throttler, query_info, res);
stream_factory.createForShard(shard_info, query, query_ast, new_context, throttler, query_info, plans, remote_pipes, delayed_pipes);
return Pipe::unitePipes(std::move(res));
if (!remote_pipes.empty())
{
auto plan = std::make_unique<QueryPlan>();
auto read_from_remote = std::make_unique<ReadFromPreparedSource>(Pipe::unitePipes(std::move(remote_pipes)));
read_from_remote->setStepDescription("Read from remote replica");
plan->addStep(std::move(read_from_remote));
plans.emplace_back(std::move(plan));
}
if (!delayed_pipes.empty())
{
auto plan = std::make_unique<QueryPlan>();
auto read_from_remote = std::make_unique<ReadFromPreparedSource>(Pipe::unitePipes(std::move(delayed_pipes)));
read_from_remote->setStepDescription("Read from delayed local replica");
plan->addStep(std::move(read_from_remote));
plans.emplace_back(std::move(plan));
}
if (plans.empty())
return;
if (plans.size() == 1)
{
query_plan = std::move(*plans.front());
return;
}
DataStreams input_streams;
input_streams.reserve(plans.size());
for (auto & plan : plans)
input_streams.emplace_back(plan->getCurrentDataStream());
auto header = input_streams.front().header;
auto union_step = std::make_unique<UnionStep>(std::move(input_streams), header);
query_plan.unitePlans(std::move(union_step), std::move(plans));
}
}

7
src/Interpreters/ClusterProxy/executeQuery.h
View File

@ -11,6 +11,7 @@ class Cluster;
struct SelectQueryInfo;
class Pipe;
class QueryPlan;
namespace ClusterProxy
{
@ -31,8 +32,10 @@ Context updateSettingsForCluster(const Cluster & cluster, const Context & contex
/// Execute a distributed query, creating a vector of BlockInputStreams, from which the result can be read.
/// `stream_factory` object encapsulates the logic of creating streams for a different type of query
/// (currently SELECT, DESCRIBE).
Pipe executeQuery(
IStreamFactory & stream_factory, Poco::Logger * log, const ASTPtr & query_ast, const Context & context, const SelectQueryInfo & query_info);
void executeQuery(
QueryPlan & query_plan,
IStreamFactory & stream_factory, Poco::Logger * log,
const ASTPtr & query_ast, const Context & context, const SelectQueryInfo & query_info);
}

82
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -33,11 +33,13 @@
#include <Processors/Pipe.h>
#include <Processors/Sources/SourceFromInputStream.h>
#include <Processors/Sources/NullSource.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <Processors/Transforms/JoiningTransform.h>
#include <Processors/Transforms/AggregatingTransform.h>
#include <Processors/Transforms/FilterTransform.h>
#include <Processors/QueryPlan/ArrayJoinStep.h>
#include <Processors/QueryPlan/ReadFromStorageStep.h>
#include <Processors/QueryPlan/SettingQuotaAndLimitsStep.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/FilterStep.h>
#include <Processors/QueryPlan/ReadNothingStep.h>
@ -1108,6 +1110,48 @@ static StreamLocalLimits getLimitsForStorage(const Settings & settings, const Se
return limits;
}
void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, const Block & source_header, const SelectQueryInfo & query_info)
{
Pipe pipe(std::make_shared<NullSource>(source_header));
if (query_info.prewhere_info)
{
if (query_info.prewhere_info->alias_actions)
{
pipe.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ExpressionTransform>(header, query_info.prewhere_info->alias_actions);
});
}
pipe.addSimpleTransform([&](const Block & header)
{
return std::make_shared<FilterTransform>(
header,
query_info.prewhere_info->prewhere_actions,
query_info.prewhere_info->prewhere_column_name,
query_info.prewhere_info->remove_prewhere_column);
});
// To remove additional columns
// In some cases, we did not read any marks so that the pipeline.streams is empty
// Thus, some columns in prewhere are not removed as expected
// This leads to mismatched header in distributed table
if (query_info.prewhere_info->remove_columns_actions)
{
pipe.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ExpressionTransform>(
header, query_info.prewhere_info->remove_columns_actions);
});
}
}
auto read_from_pipe = std::make_unique<ReadFromPreparedSource>(std::move(pipe));
read_from_pipe->setStepDescription("Read from NullSource");
query_plan.addStep(std::move(read_from_pipe));
}
void InterpreterSelectQuery::executeFetchColumns(
QueryProcessingStage::Enum processing_stage, QueryPlan & query_plan,
const PrewhereInfoPtr & prewhere_info, const Names & columns_to_remove_after_prewhere)
@ -1353,7 +1397,7 @@ void InterpreterSelectQuery::executeFetchColumns(
ErrorCodes::TOO_MANY_COLUMNS);
/// General limit for the number of threads.
query_plan.setMaxThreads(settings.max_threads);
size_t max_threads_execute_query = settings.max_threads;
/** With distributed query processing, almost no computations are done in the threads,
* but wait and receive data from remote servers.
@ -1366,8 +1410,7 @@ void InterpreterSelectQuery::executeFetchColumns(
if (storage && storage->isRemote())
{
is_remote = true;
max_streams = settings.max_distributed_connections;
query_plan.setMaxThreads(max_streams);
max_threads_execute_query = max_streams = settings.max_distributed_connections;
}
UInt64 max_block_size = settings.max_block_size;
@ -1392,8 +1435,7 @@ void InterpreterSelectQuery::executeFetchColumns(
&& limit_length + limit_offset < max_block_size)
{
max_block_size = std::max(UInt64(1), limit_length + limit_offset);
max_streams = 1;
query_plan.setMaxThreads(max_streams);
max_threads_execute_query = max_streams = 1;
}
if (!max_block_size)
@ -1473,12 +1515,36 @@ void InterpreterSelectQuery::executeFetchColumns(
if (!options.ignore_quota && (options.to_stage == QueryProcessingStage::Complete))
quota = context->getQuota();
storage->read(query_plan, table_lock, metadata_snapshot, limits, leaf_limits, std::move(quota),
required_columns, query_info, context, processing_stage, max_block_size, max_streams);
storage->read(query_plan, required_columns, metadata_snapshot,
query_info, *context, processing_stage, max_block_size, max_streams);
/// Create step which reads from empty source if storage has no data.
if (!query_plan.isInitialized())
{
auto header = metadata_snapshot->getSampleBlockForColumns(
required_columns, storage->getVirtuals(), storage->getStorageID());
addEmptySourceToQueryPlan(query_plan, header, query_info);
}
/// Extend lifetime of context, table lock, storage. Set limits and quota.
auto adding_limits_and_quota = std::make_unique<SettingQuotaAndLimitsStep>(
query_plan.getCurrentDataStream(),
storage,
std::move(table_lock),
limits,
leaf_limits,
std::move(quota),
context);
adding_limits_and_quota->setStepDescription("Set limits and quota after reading from storage");
query_plan.addStep(std::move(adding_limits_and_quota));
}
else
throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR);
/// Specify the number of threads only if it wasn't specified in storage.
if (!query_plan.getMaxThreads())
query_plan.setMaxThreads(max_threads_execute_query);
/// Aliases in table declaration.
if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions)
{

2
src/Interpreters/InterpreterSelectQuery.h
View File

@ -94,6 +94,8 @@ public:
const SelectQueryInfo & getQueryInfo() const { return query_info; }
static void addEmptySourceToQueryPlan(QueryPlan & query_plan, const Block & source_header, const SelectQueryInfo & query_info);
private:
InterpreterSelectQuery(
const ASTPtr & query_ptr_,

100
src/Interpreters/MutationsInterpreter.cpp
View File

@ -5,6 +5,7 @@
#include <Interpreters/MutationsInterpreter.h>
#include <Interpreters/TreeRewriter.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/MergeTree/StorageFromMergeTreeDataPart.h>
#include <Processors/Transforms/FilterTransform.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <Processors/Transforms/CreatingSetsTransform.h>
@ -32,6 +33,7 @@ namespace DB
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
extern const int UNKNOWN_MUTATION_COMMAND;
@ -92,6 +94,7 @@ std::optional<String> findFirstNonDeterministicFunctionName(const MutationComman
if (finder_data.nondeterministic_function_name)
return finder_data.nondeterministic_function_name;
/// Currently UPDATE and DELETE both always have predicates so we can use fallthrough
[[fallthrough]];
}
@ -110,7 +113,7 @@ std::optional<String> findFirstNonDeterministicFunctionName(const MutationComman
return {};
}
ASTPtr prepareQueryAffectedAST(const std::vector<MutationCommand> & commands)
ASTPtr prepareQueryAffectedAST(const std::vector<MutationCommand> & commands, const StoragePtr & storage, const Context & context)
{
/// Execute `SELECT count() FROM storage WHERE predicate1 OR predicate2 OR ...` query.
/// The result can differ from the number of affected rows (e.g. if there is an UPDATE command that
@ -125,20 +128,23 @@ ASTPtr prepareQueryAffectedAST(const std::vector<MutationCommand> & commands)
count_func->arguments = std::make_shared<ASTExpressionList>();
select->select()->children.push_back(count_func);
if (commands.size() == 1)
select->setExpression(ASTSelectQuery::Expression::WHERE, commands[0].predicate->clone());
else
ASTs conditions;
for (const MutationCommand & command : commands)
{
auto coalesced_predicates = std::make_shared<ASTFunction>();
coalesced_predicates->name = "or";
coalesced_predicates->arguments = std::make_shared<ASTExpressionList>();
coalesced_predicates->children.push_back(coalesced_predicates->arguments);
for (const MutationCommand & command : commands)
coalesced_predicates->arguments->children.push_back(command.predicate->clone());
if (ASTPtr condition = getPartitionAndPredicateExpressionForMutationCommand(command, storage, context))
conditions.push_back(std::move(condition));
}
if (conditions.size() > 1)
{
auto coalesced_predicates = makeASTFunction("or");
coalesced_predicates->arguments->children = std::move(conditions);
select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(coalesced_predicates));
}
else if (conditions.size() == 1)
{
select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(conditions.front()));
}
return select;
}
@ -167,8 +173,9 @@ ColumnDependencies getAllColumnDependencies(const StorageMetadataPtr & metadata_
}
bool isStorageTouchedByMutations(
StoragePtr storage,
const StoragePtr & storage,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MutationCommand> & commands,
Context context_copy)
@ -176,16 +183,33 @@ bool isStorageTouchedByMutations(
if (commands.empty())
return false;
bool all_commands_can_be_skipped = true;
auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast<StorageFromMergeTreeDataPart>(storage);
for (const MutationCommand & command : commands)
{
if (!command.predicate) /// The command touches all rows.
return true;
if (command.partition && !storage_from_merge_tree_data_part)
throw Exception("ALTER UPDATE/DELETE ... IN PARTITION is not supported for non-MergeTree tables", ErrorCodes::NOT_IMPLEMENTED);
if (command.partition && storage_from_merge_tree_data_part)
{
const String partition_id = storage_from_merge_tree_data_part->getPartitionIDFromQuery(command.partition, context_copy);
if (partition_id == storage_from_merge_tree_data_part->getPartitionId())
all_commands_can_be_skipped = false;
}
else
all_commands_can_be_skipped = false;
}
if (all_commands_can_be_skipped)
return false;
context_copy.setSetting("max_streams_to_max_threads_ratio", 1);
context_copy.setSetting("max_threads", 1);
ASTPtr select_query = prepareQueryAffectedAST(commands);
ASTPtr select_query = prepareQueryAffectedAST(commands, storage, context_copy);
/// Interpreter must be alive, when we use result of execute() method.
/// For some reason it may copy context and and give it into ExpressionBlockInputStream
@ -202,9 +226,42 @@ bool isStorageTouchedByMutations(
auto count = (*block.getByName("count()").column)[0].get<UInt64>();
return count != 0;
}
ASTPtr getPartitionAndPredicateExpressionForMutationCommand(
const MutationCommand & command,
const StoragePtr & storage,
const Context & context
)
{
ASTPtr partition_predicate_as_ast_func;
if (command.partition)
{
String partition_id;
auto storage_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(storage);
auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast<StorageFromMergeTreeDataPart>(storage);
if (storage_merge_tree)
partition_id = storage_merge_tree->getPartitionIDFromQuery(command.partition, context);
else if (storage_from_merge_tree_data_part)
partition_id = storage_from_merge_tree_data_part->getPartitionIDFromQuery(command.partition, context);
else
throw Exception("ALTER UPDATE/DELETE ... IN PARTITION is not supported for non-MergeTree tables", ErrorCodes::NOT_IMPLEMENTED);
partition_predicate_as_ast_func = makeASTFunction("equals",
std::make_shared<ASTIdentifier>("_partition_id"),
std::make_shared<ASTLiteral>(partition_id)
);
}
if (command.predicate && command.partition)
return makeASTFunction("and", command.predicate->clone(), std::move(partition_predicate_as_ast_func));
else
return command.predicate ? command.predicate->clone() : partition_predicate_as_ast_func;
}
MutationsInterpreter::MutationsInterpreter(
StoragePtr storage_,
const StorageMetadataPtr & metadata_snapshot_,
@ -349,7 +406,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
if (stages.empty() || !stages.back().column_to_updated.empty())
stages.emplace_back(context);
auto negated_predicate = makeASTFunction("isZeroOrNull", command.predicate->clone());
auto negated_predicate = makeASTFunction("isZeroOrNull", getPartitionAndPredicateExpressionForMutationCommand(command));
stages.back().filters.push_back(negated_predicate);
}
else if (command.type == MutationCommand::UPDATE)
@ -387,7 +444,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
const auto & update_expr = kv.second;
auto updated_column = makeASTFunction("CAST",
makeASTFunction("if",
command.predicate->clone(),
getPartitionAndPredicateExpressionForMutationCommand(command),
makeASTFunction("CAST",
update_expr->clone(),
type_literal),
@ -592,7 +649,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector<Stage> &
for (const String & column : stage.output_columns)
all_asts->children.push_back(std::make_shared<ASTIdentifier>(column));
auto syntax_result = TreeRewriter(context).analyze(all_asts, all_columns);
auto syntax_result = TreeRewriter(context).analyze(all_asts, all_columns, storage, metadata_snapshot);
if (context.hasQueryContext())
for (const auto & it : syntax_result->getScalars())
context.getQueryContext().addScalar(it.first, it.second);
@ -759,10 +816,10 @@ const Block & MutationsInterpreter::getUpdatedHeader() const
size_t MutationsInterpreter::evaluateCommandsSize()
{
for (const MutationCommand & command : commands)
if (unlikely(!command.predicate)) /// The command touches all rows.
if (unlikely(!command.predicate && !command.partition)) /// The command touches all rows.
return mutation_ast->size();
return std::max(prepareQueryAffectedAST(commands)->size(), mutation_ast->size());
return std::max(prepareQueryAffectedAST(commands, storage, context)->size(), mutation_ast->size());
}
std::optional<SortDescription> MutationsInterpreter::getStorageSortDescriptionIfPossible(const Block & header) const
@ -783,6 +840,11 @@ std::optional<SortDescription> MutationsInterpreter::getStorageSortDescriptionIf
return sort_description;
}
ASTPtr MutationsInterpreter::getPartitionAndPredicateExpressionForMutationCommand(const MutationCommand & command) const
{
return DB::getPartitionAndPredicateExpressionForMutationCommand(command, storage, context);
}
bool MutationsInterpreter::Stage::isAffectingAllColumns(const Names & storage_columns) const
{
/// is subset

14
src/Interpreters/MutationsInterpreter.h
View File

@ -20,7 +20,17 @@ using QueryPipelinePtr = std::unique_ptr<QueryPipeline>;
/// Return false if the data isn't going to be changed by mutations.
bool isStorageTouchedByMutations(
StoragePtr storage, const StorageMetadataPtr & metadata_snapshot, const std::vector<MutationCommand> & commands, Context context_copy);
const StoragePtr & storage,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MutationCommand> & commands,
Context context_copy
);
ASTPtr getPartitionAndPredicateExpressionForMutationCommand(
const MutationCommand & command,
const StoragePtr & storage,
const Context & context
);
/// Create an input stream that will read data from storage and apply mutation commands (UPDATEs, DELETEs, MATERIALIZEs)
/// to this data.
@ -59,6 +69,8 @@ private:
std::optional<SortDescription> getStorageSortDescriptionIfPossible(const Block & header) const;
ASTPtr getPartitionAndPredicateExpressionForMutationCommand(const MutationCommand & command) const;
StoragePtr storage;
StorageMetadataPtr metadata_snapshot;
MutationCommands commands;

4
src/Interpreters/SystemLog.cpp
View File

@ -25,6 +25,7 @@ namespace
{
constexpr size_t DEFAULT_SYSTEM_LOG_FLUSH_INTERVAL_MILLISECONDS = 7500;
constexpr size_t DEFAULT_METRIC_LOG_COLLECT_INTERVAL_MILLISECONDS = 1000;
/// Creates a system log with MergeTree engine using parameters from config
template <typename TSystemLog>
@ -125,7 +126,8 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi
if (metric_log)
{
size_t collect_interval_milliseconds = config.getUInt64("metric_log.collect_interval_milliseconds");
size_t collect_interval_milliseconds = config.getUInt64("metric_log.collect_interval_milliseconds",
DEFAULT_METRIC_LOG_COLLECT_INTERVAL_MILLISECONDS);
metric_log->startCollectMetric(collect_interval_milliseconds);
}

6
src/Interpreters/TreeRewriter.cpp
View File

@ -63,7 +63,7 @@ struct CustomizeFunctionsData
const String & customized_func_name;
void visit(ASTFunction & func, ASTPtr &)
void visit(ASTFunction & func, ASTPtr &) const
{
if (Poco::toLower(func.name) == func_name)
{
@ -97,7 +97,7 @@ struct CustomizeFunctionsSuffixData
const String & customized_func_suffix;
void visit(ASTFunction & func, ASTPtr &)
void visit(ASTFunction & func, ASTPtr &) const
{
if (endsWith(Poco::toLower(func.name), func_suffix))
{
@ -118,7 +118,7 @@ struct CustomizeAggregateFunctionsSuffixData
const String & customized_func_suffix;
void visit(ASTFunction & func, ASTPtr &)
void visit(ASTFunction & func, ASTPtr &) const
{
const auto & instance = AggregateFunctionFactory::instance();
if (instance.isAggregateFunctionName(func.name) && !endsWith(func.name, customized_func_suffix))

24
src/Parsers/ASTAlterQuery.cpp
View File

@ -90,7 +90,7 @@ void ASTAlterCommand::formatImpl(
column->formatImpl(settings, state, frame);
if (partition)
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : "");
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : "");
partition->formatImpl(settings, state, frame);
}
}
@ -150,7 +150,7 @@ void ASTAlterCommand::formatImpl(
index->formatImpl(settings, state, frame);
if (partition)
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : "");
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : "");
partition->formatImpl(settings, state, frame);
}
}
@ -161,7 +161,7 @@ void ASTAlterCommand::formatImpl(
index->formatImpl(settings, state, frame);
if (partition)
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : "");
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : "");
partition->formatImpl(settings, state, frame);
}
}
@ -272,7 +272,15 @@ void ASTAlterCommand::formatImpl(
}
else if (type == ASTAlterCommand::DELETE)
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DELETE WHERE " << (settings.hilite ? hilite_none : "");
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DELETE" << (settings.hilite ? hilite_none : "");
if (partition)
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : "");
partition->formatImpl(settings, state, frame);
}
settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : "");
predicate->formatImpl(settings, state, frame);
}
else if (type == ASTAlterCommand::UPDATE)
@ -280,6 +288,12 @@ void ASTAlterCommand::formatImpl(
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "UPDATE " << (settings.hilite ? hilite_none : "");
update_assignments->formatImpl(settings, state, frame);
if (partition)
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : "");
partition->formatImpl(settings, state, frame);
}
settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : "");
predicate->formatImpl(settings, state, frame);
}
@ -298,7 +312,7 @@ void ASTAlterCommand::formatImpl(
<< (settings.hilite ? hilite_none : "");
if (partition)
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : "");
settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : "");
partition->formatImpl(settings, state, frame);
}
}

2
src/Parsers/ASTAlterQuery.h
View File

@ -103,7 +103,7 @@ public:
*/
ASTPtr constraint;
/** Used in DROP PARTITION and ATTACH PARTITION FROM queries.
/** Used in DROP PARTITION, ATTACH PARTITION FROM, UPDATE, DELETE queries.
* The value or ID of the partition is stored here.
*/
ASTPtr partition;

14
src/Parsers/ExpressionListParsers.cpp
View File

@ -55,6 +55,12 @@ const char * ParserComparisonExpression::operators[] =
nullptr
};
const char * ParserComparisonExpression::overlapping_operators_to_skip[] =
{
"IN PARTITION",
nullptr
};
const char * ParserLogicalNotExpression::operators[] =
{
"NOT", "not",
@ -137,6 +143,14 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node
/// try to find any of the valid operators
const char ** it;
Expected stub;
for (it = overlapping_operators_to_skip; *it; ++it)
if (ParserKeyword{*it}.checkWithoutMoving(pos, stub))
break;
if (*it)
break;
for (it = operators; *it; it += 2)
if (parseOperator(pos, *it, expected))
break;

9
src/Parsers/ExpressionListParsers.h
View File

@ -82,6 +82,7 @@ class ParserLeftAssociativeBinaryOperatorList : public IParserBase
{
private:
Operators_t operators;
Operators_t overlapping_operators_to_skip = { (const char *[]){ nullptr } };
ParserPtr first_elem_parser;
ParserPtr remaining_elem_parser;
@ -93,6 +94,11 @@ public:
{
}
ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, Operators_t overlapping_operators_to_skip_, ParserPtr && first_elem_parser_)
: operators(operators_), overlapping_operators_to_skip(overlapping_operators_to_skip_), first_elem_parser(std::move(first_elem_parser_))
{
}
ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, ParserPtr && first_elem_parser_,
ParserPtr && remaining_elem_parser_)
: operators(operators_), first_elem_parser(std::move(first_elem_parser_)),
@ -284,7 +290,8 @@ class ParserComparisonExpression : public IParserBase
{
private:
static const char * operators[];
ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique<ParserBetweenExpression>()};
static const char * overlapping_operators_to_skip[];
ParserLeftAssociativeBinaryOperatorList operator_parser {operators, overlapping_operators_to_skip, std::make_unique<ParserBetweenExpression>()};
protected:
const char * getName() const override{ return "comparison expression"; }

19
src/Parsers/ParserAlterQuery.cpp
View File

@ -79,7 +79,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
ParserKeyword s_to_volume("TO VOLUME");
ParserKeyword s_to_table("TO TABLE");
ParserKeyword s_delete_where("DELETE WHERE");
ParserKeyword s_delete("DELETE");
ParserKeyword s_update("UPDATE");
ParserKeyword s_where("WHERE");
ParserKeyword s_to("TO");
@ -506,8 +506,17 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
command->type = ASTAlterCommand::MODIFY_SAMPLE_BY;
}
else if (s_delete_where.ignore(pos, expected))
else if (s_delete.ignore(pos, expected))
{
if (s_in_partition.ignore(pos, expected))
{
if (!parser_partition.parse(pos, command->partition, expected))
return false;
}
if (!s_where.ignore(pos, expected))
return false;
if (!parser_exp_elem.parse(pos, command->predicate, expected))
return false;
@ -518,6 +527,12 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
if (!parser_assignment_list.parse(pos, command->update_assignments, expected))
return false;
if (s_in_partition.ignore(pos, expected))
{
if (!parser_partition.parse(pos, command->partition, expected))
return false;
}
if (!s_where.ignore(pos, expected))
return false;

10
src/Parsers/ParserAlterQuery.h
View File

@ -10,7 +10,7 @@ namespace DB
* ALTER TABLE [db.]name [ON CLUSTER cluster]
* [ADD COLUMN [IF NOT EXISTS] col_name type [AFTER col_after],]
* [DROP COLUMN [IF EXISTS] col_to_drop, ...]
* [CLEAR COLUMN [IF EXISTS] col_to_clear [IN PARTITION partition],]
* [CLEAR COLUMN [IF EXISTS] col_to_clear[ IN PARTITION partition],]
* [MODIFY COLUMN [IF EXISTS] col_to_modify type, ...]
* [RENAME COLUMN [IF EXISTS] col_name TO col_name]
* [MODIFY PRIMARY KEY (a, b, c...)]
@ -19,8 +19,12 @@ namespace DB
* [DROP|DETACH|ATTACH PARTITION|PART partition, ...]
* [FETCH PARTITION partition FROM ...]
* [FREEZE [PARTITION] [WITH NAME name]]
* [DELETE WHERE ...]
* [UPDATE col_name = expr, ... WHERE ...]
* [DELETE[ IN PARTITION partition] WHERE ...]
* [UPDATE col_name = expr, ...[ IN PARTITION partition] WHERE ...]
* [ADD INDEX [IF NOT EXISTS] index_name [AFTER index_name]]
* [DROP INDEX [IF EXISTS] index_name]
* [CLEAR INDEX [IF EXISTS] index_name IN PARTITION partition]
* [MATERIALIZE INDEX [IF EXISTS] index_name [IN PARTITION partition]]
* ALTER LIVE VIEW [db.name]
* [REFRESH]
*/

6
src/Processors/QueryPipeline.cpp
View File

@ -95,6 +95,12 @@ void QueryPipeline::addTransform(ProcessorPtr transform)
pipe.addTransform(std::move(transform));
}
void QueryPipeline::transform(const Transformer & transformer)
{
checkInitializedAndNotCompleted();
pipe.transform(transformer);
}
void QueryPipeline::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter)
{
checkInitializedAndNotCompleted();

8
src/Processors/QueryPipeline.h
View File

@ -53,6 +53,11 @@ public:
void addSimpleTransform(const Pipe::ProcessorGetterWithStreamKind & getter);
/// Add transform with getNumStreams() input ports.
void addTransform(ProcessorPtr transform);
using Transformer = std::function<Processors(OutputPortRawPtrs ports)>;
/// Transform pipeline in general way.
void transform(const Transformer & transformer);
/// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port.
void addTotalsHavingTransform(ProcessorPtr transform);
/// Add transform which calculates extremes. This transform adds extremes port and doesn't change inputs number.
@ -105,6 +110,9 @@ public:
void addInterpreterContext(std::shared_ptr<Context> context) { pipe.addInterpreterContext(std::move(context)); }
void addStorageHolder(StoragePtr storage) { pipe.addStorageHolder(std::move(storage)); }
void addQueryPlan(std::unique_ptr<QueryPlan> plan) { pipe.addQueryPlan(std::move(plan)); }
void setLimits(const StreamLocalLimits & limits) { pipe.setLimits(limits); }
void setLeafLimits(const SizeLimits & limits) { pipe.setLeafLimits(limits); }
void setQuota(const std::shared_ptr<const EnabledQuota> & quota) { pipe.setQuota(quota); }
/// For compatibility with IBlockInputStream.
void setProgressCallback(const ProgressCallback & callback);

41
src/Processors/QueryPlan/AddingConstColumnStep.cpp Normal file
View File

@ -0,0 +1,41 @@
#include <Processors/QueryPlan/AddingConstColumnStep.h>
#include <Processors/QueryPipeline.h>
#include <Processors/Transforms/AddingConstColumnTransform.h>
#include <IO/Operators.h>
namespace DB
{
static ITransformingStep::Traits getTraits()
{
return ITransformingStep::Traits
{
{
.preserves_distinct_columns = true,
.returns_single_stream = false,
.preserves_number_of_streams = true,
.preserves_sorting = true,
},
{
.preserves_number_of_rows = true,
}
};
}
AddingConstColumnStep::AddingConstColumnStep(const DataStream & input_stream_, ColumnWithTypeAndName column_)
: ITransformingStep(input_stream_,
AddingConstColumnTransform::transformHeader(input_stream_.header, column_),
getTraits())
, column(std::move(column_))
{
}
void AddingConstColumnStep::transformPipeline(QueryPipeline & pipeline)
{
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<AddingConstColumnTransform>(header, column);
});
}
}

22
src/Processors/QueryPlan/AddingConstColumnStep.h Normal file
View File

@ -0,0 +1,22 @@
#pragma once
#include <Processors/QueryPlan/ITransformingStep.h>
namespace DB
{
/// Adds a materialized const column with a specified value.
class AddingConstColumnStep : public ITransformingStep
{
public:
AddingConstColumnStep(const DataStream & input_stream_, ColumnWithTypeAndName column_);
String getName() const override { return "AddingConstColumn"; }
void transformPipeline(QueryPipeline & pipeline) override;
private:
ColumnWithTypeAndName column;
};
}

45
src/Processors/QueryPlan/AddingMissedStep.cpp Normal file
View File

@ -0,0 +1,45 @@
#include <Processors/QueryPlan/AddingMissedStep.h>
#include <Processors/QueryPipeline.h>
#include <Processors/Transforms/AddingMissedTransform.h>
#include <IO/Operators.h>
namespace DB
{
static ITransformingStep::Traits getTraits()
{
return ITransformingStep::Traits
{
{
.preserves_distinct_columns = true,
.returns_single_stream = false,
.preserves_number_of_streams = true,
.preserves_sorting = true,
},
{
.preserves_number_of_rows = true,
}
};
}
AddingMissedStep::AddingMissedStep(
const DataStream & input_stream_,
Block result_header_,
ColumnsDescription columns_,
const Context & context_)
: ITransformingStep(input_stream_, result_header_, getTraits())
, columns(std::move(columns_))
, context(context_)
{
updateDistinctColumns(output_stream->header, output_stream->distinct_columns);
}
void AddingMissedStep::transformPipeline(QueryPipeline & pipeline)
{
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<AddingMissedTransform>(header, output_stream->header, columns, context);
});
}
}

26
src/Processors/QueryPlan/AddingMissedStep.h Normal file
View File

@ -0,0 +1,26 @@
#pragma once
#include <Processors/QueryPlan/ITransformingStep.h>
#include <Storages/ColumnsDescription.h>
namespace DB
{
/// Convert one block structure to another. See ConvertingTransform.
class AddingMissedStep : public ITransformingStep
{
public:
AddingMissedStep(const DataStream & input_stream_,
Block result_header_,
ColumnsDescription columns_,
const Context & context_);
String getName() const override { return "AddingMissed"; }
void transformPipeline(QueryPipeline & pipeline) override;
private:
ColumnsDescription columns;
const Context & context;
};
}

3
src/Processors/QueryPlan/ISourceStep.cpp
View File

@ -14,7 +14,8 @@ QueryPipelinePtr ISourceStep::updatePipeline(QueryPipelines)
auto pipeline = std::make_unique<QueryPipeline>();
QueryPipelineProcessorsCollector collector(*pipeline, this);
initializePipeline(*pipeline);
processors = collector.detachProcessors();
auto added_processors = collector.detachProcessors();
processors.insert(processors.end(), added_processors.begin(), added_processors.end());
return pipeline;
}

2
src/Processors/QueryPlan/ISourceStep.h
View File

@ -16,7 +16,7 @@ public:
void describePipeline(FormatSettings & settings) const override;
private:
protected:
/// We collect processors got after pipeline transformation.
Processors processors;
};

39
src/Processors/QueryPlan/MaterializingStep.cpp Normal file
View File

@ -0,0 +1,39 @@
#include <Processors/QueryPlan/MaterializingStep.h>
#include <Processors/QueryPipeline.h>
#include <Processors/Transforms/MaterializingTransform.h>
#include <DataStreams/materializeBlock.h>
namespace DB
{
static ITransformingStep::Traits getTraits()
{
return ITransformingStep::Traits
{
{
.preserves_distinct_columns = true,
.returns_single_stream = false,
.preserves_number_of_streams = true,
.preserves_sorting = true,
},
{
.preserves_number_of_rows = true,
}
};
}
MaterializingStep::MaterializingStep(const DataStream & input_stream_)
: ITransformingStep(input_stream_, materializeBlock(input_stream_.header), getTraits())
{
}
void MaterializingStep::transformPipeline(QueryPipeline & pipeline)
{
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<MaterializingTransform>(header);
});
}
}

18
src/Processors/QueryPlan/MaterializingStep.h Normal file
View File

@ -0,0 +1,18 @@
#pragma once
#include <Processors/QueryPlan/ITransformingStep.h>
namespace DB
{
/// Materialize constants. See MaterializingTransform.
class MaterializingStep : public ITransformingStep
{
public:
explicit MaterializingStep(const DataStream & input_stream_);
String getName() const override { return "Materializing"; }
void transformPipeline(QueryPipeline & pipeline) override;
};
}

164
src/Processors/QueryPlan/MergingFinal.cpp Normal file
View File

@ -0,0 +1,164 @@
#include <Processors/QueryPlan/MergingFinal.h>
#include <Processors/QueryPipeline.h>
#include <Processors/Merges/AggregatingSortedTransform.h>
#include <Processors/Merges/CollapsingSortedTransform.h>
#include <Processors/Merges/MergingSortedTransform.h>
#include <Processors/Merges/ReplacingSortedTransform.h>
#include <Processors/Merges/SummingSortedTransform.h>
#include <Processors/Merges/VersionedCollapsingTransform.h>
#include <Processors/Transforms/AddingSelectorTransform.h>
#include <Processors/Transforms/CopyTransform.h>
#include <IO/Operators.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
static ITransformingStep::Traits getTraits()
{
return ITransformingStep::Traits
{
{
.preserves_distinct_columns = true,
.returns_single_stream = false,
.preserves_number_of_streams = false,
.preserves_sorting = false,
},
{
.preserves_number_of_rows = true,
}
};
}
MergingFinal::MergingFinal(
const DataStream & input_stream,
size_t num_output_streams_,
SortDescription sort_description_,
MergeTreeData::MergingParams params_,
Names partition_key_columns_,
size_t max_block_size_)
: ITransformingStep(input_stream, input_stream.header, getTraits())
, num_output_streams(num_output_streams_)
, sort_description(std::move(sort_description_))
, merging_params(std::move(params_))
, partition_key_columns(std::move(partition_key_columns_))
, max_block_size(max_block_size_)
{
/// TODO: check input_stream is partially sorted (each port) by the same description.
// output_stream->sort_description = sort_description;
// output_stream->sort_mode = DataStream::SortMode::Stream;
}
void MergingFinal::transformPipeline(QueryPipeline & pipeline)
{
const auto & header = pipeline.getHeader();
size_t num_outputs = pipeline.getNumStreams();
auto get_merging_processor = [&]() -> MergingTransformPtr
{
switch (merging_params.mode)
{
case MergeTreeData::MergingParams::Ordinary:
{
return std::make_shared<MergingSortedTransform>(header, num_outputs,
sort_description, max_block_size);
}
case MergeTreeData::MergingParams::Collapsing:
return std::make_shared<CollapsingSortedTransform>(header, num_outputs,
sort_description, merging_params.sign_column, true, max_block_size);
case MergeTreeData::MergingParams::Summing:
return std::make_shared<SummingSortedTransform>(header, num_outputs,
sort_description, merging_params.columns_to_sum, partition_key_columns, max_block_size);
case MergeTreeData::MergingParams::Aggregating:
return std::make_shared<AggregatingSortedTransform>(header, num_outputs,
sort_description, max_block_size);
case MergeTreeData::MergingParams::Replacing:
return std::make_shared<ReplacingSortedTransform>(header, num_outputs,
sort_description, merging_params.version_column, max_block_size);
case MergeTreeData::MergingParams::VersionedCollapsing:
return std::make_shared<VersionedCollapsingTransform>(header, num_outputs,
sort_description, merging_params.sign_column, max_block_size);
case MergeTreeData::MergingParams::Graphite:
throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR);
}
__builtin_unreachable();
};
if (num_output_streams <= 1 || sort_description.empty())
{
pipeline.addTransform(get_merging_processor());
return;
}
ColumnNumbers key_columns;
key_columns.reserve(sort_description.size());
for (auto & desc : sort_description)
{
if (!desc.column_name.empty())
key_columns.push_back(header.getPositionByName(desc.column_name));
else
key_columns.emplace_back(desc.column_number);
}
pipeline.addSimpleTransform([&](const Block & stream_header)
{
return std::make_shared<AddingSelectorTransform>(stream_header, num_output_streams, key_columns);
});
pipeline.transform([&](OutputPortRawPtrs ports)
{
Processors transforms;
std::vector<OutputPorts::iterator> output_ports;
transforms.reserve(ports.size() + num_output_streams);
output_ports.reserve(ports.size());
for (auto & port : ports)
{
auto copier = std::make_shared<CopyTransform>(header, num_output_streams);
connect(*port, copier->getInputPort());
output_ports.emplace_back(copier->getOutputs().begin());
transforms.emplace_back(std::move(copier));
}
for (size_t i = 0; i < num_output_streams; ++i)
{
auto merge = get_merging_processor();
merge->setSelectorPosition(i);
auto input = merge->getInputs().begin();
/// Connect i-th merge with i-th input port of every copier.
for (size_t j = 0; j < ports.size(); ++j)
{
connect(*output_ports[j], *input);
++output_ports[j];
++input;
}
transforms.emplace_back(std::move(merge));
}
return transforms;
});
}
void MergingFinal::describeActions(FormatSettings & settings) const
{
String prefix(settings.offset, ' ');
settings.out << prefix << "Sort description: ";
dumpSortDescription(sort_description, input_streams.front().header, settings.out);
settings.out << '\n';
}
}

35
src/Processors/QueryPlan/MergingFinal.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#include <Processors/QueryPlan/ITransformingStep.h>
#include <Core/SortDescription.h>
#include <Storages/MergeTree/MergeTreeData.h>
namespace DB
{
/// Merge streams of data into single sorted stream.
class MergingFinal : public ITransformingStep
{
public:
explicit MergingFinal(
const DataStream & input_stream,
size_t num_output_streams_,
SortDescription sort_description_,
MergeTreeData::MergingParams params_,
Names partition_key_columns_,
size_t max_block_size_);
String getName() const override { return "MergingFinal"; }
void transformPipeline(QueryPipeline & pipeline) override;
void describeActions(FormatSettings & settings) const override;
private:
size_t num_output_streams;
SortDescription sort_description;
MergeTreeData::MergingParams merging_params;
Names partition_key_columns;
size_t max_block_size;
};
}

11
src/Processors/QueryPlan/QueryPlan.cpp
View File

@ -185,6 +185,17 @@ QueryPipelinePtr QueryPlan::buildQueryPipeline()
return last_pipeline;
}
Pipe QueryPlan::convertToPipe()
{
if (!isInitialized())
return {};
if (isCompleted())
throw Exception("Cannot convert completed QueryPlan to Pipe", ErrorCodes::LOGICAL_ERROR);
return QueryPipeline::getPipe(std::move(*buildQueryPipeline()));
}
void QueryPlan::addInterpreterContext(std::shared_ptr<Context> context)
{
interpreter_context.emplace_back(std::move(context));

11
src/Processors/QueryPlan/QueryPlan.h
View File

@ -17,6 +17,11 @@ using QueryPipelinePtr = std::unique_ptr<QueryPipeline>;
class Context;
class WriteBuffer;
class QueryPlan;
using QueryPlanPtr = std::unique_ptr<QueryPlan>;
class Pipe;
/// A tree of query steps.
/// The goal of QueryPlan is to build QueryPipeline.
/// QueryPlan let delay pipeline creation which is helpful for pipeline-level optimisations.
@ -28,7 +33,7 @@ public:
QueryPlan(QueryPlan &&);
QueryPlan & operator=(QueryPlan &&);
void unitePlans(QueryPlanStepPtr step, std::vector<std::unique_ptr<QueryPlan>> plans);
void unitePlans(QueryPlanStepPtr step, std::vector<QueryPlanPtr> plans);
void addStep(QueryPlanStepPtr step);
bool isInitialized() const { return root != nullptr; } /// Tree is not empty
@ -39,6 +44,9 @@ public:
QueryPipelinePtr buildQueryPipeline();
/// If initialized, build pipeline and convert to pipe. Otherwise, return empty pipe.
Pipe convertToPipe();
struct ExplainPlanOptions
{
/// Add output header to step.
@ -61,6 +69,7 @@ public:
/// Set upper limit for the recommend number of threads. Will be applied to the newly-created pipelines.
/// TODO: make it in a better way.
void setMaxThreads(size_t max_threads_) { max_threads = max_threads_; }
size_t getMaxThreads() const { return max_threads; }
void addInterpreterContext(std::shared_ptr<Context> context);

6
src/Processors/QueryPlan/ReadFromPreparedSource.cpp
View File

@ -5,7 +5,7 @@ namespace DB
{
ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr<Context> context_)
: ISourceStep(DataStream{.header = pipe_.getHeader(), .has_single_port = true})
: ISourceStep(DataStream{.header = pipe_.getHeader()})
, pipe(std::move(pipe_))
, context(std::move(context_))
{
@ -13,7 +13,11 @@ ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr<Conte
void ReadFromPreparedSource::initializePipeline(QueryPipeline & pipeline)
{
for (const auto & processor : pipe.getProcessors())
processors.emplace_back(processor);
pipeline.init(std::move(pipe));
if (context)
pipeline.addInterpreterContext(std::move(context));
}

14
src/Processors/QueryPlan/ReadFromPreparedSource.h
View File

@ -11,7 +11,7 @@ class ReadFromPreparedSource : public ISourceStep
public:
explicit ReadFromPreparedSource(Pipe pipe_, std::shared_ptr<Context> context_ = nullptr);
String getName() const override { return "ReadNothing"; }
String getName() const override { return "ReadFromPreparedSource"; }
void initializePipeline(QueryPipeline & pipeline) override;
@ -20,4 +20,16 @@ private:
std::shared_ptr<Context> context;
};
class ReadFromStorageStep : public ReadFromPreparedSource
{
public:
ReadFromStorageStep(Pipe pipe_, String storage_name)
: ReadFromPreparedSource(std::move(pipe_))
{
setStepDescription(storage_name);
}
String getName() const override { return "ReadFromStorage"; }
};
}

114
src/Processors/QueryPlan/ReadFromStorageStep.cpp
View File

@ -1,114 +0,0 @@
#include <Processors/QueryPlan/ReadFromStorageStep.h>
#include <Interpreters/Context.h>
#include <Processors/Sources/NullSource.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <Processors/Transforms/FilterTransform.h>
#include <Processors/Pipe.h>
#include <Processors/QueryPipeline.h>
#include <Storages/IStorage.h>
namespace DB
{
ReadFromStorageStep::ReadFromStorageStep(
TableLockHolder table_lock,
StorageMetadataPtr metadata_snapshot,
StreamLocalLimits & limits,
SizeLimits & leaf_limits,
std::shared_ptr<const EnabledQuota> quota,
StoragePtr storage,
const Names & required_columns,
SelectQueryInfo & query_info,
std::shared_ptr<Context> context,
QueryProcessingStage::Enum processing_stage,
size_t max_block_size,
size_t max_streams)
{
/// Note: we read from storage in constructor of step because we don't know real header before reading.
/// It will be fixed when storage return QueryPlanStep itself.
Pipe pipe = storage->read(required_columns, metadata_snapshot, query_info, *context, processing_stage, max_block_size, max_streams);
if (pipe.empty())
{
pipe = Pipe(std::make_shared<NullSource>(metadata_snapshot->getSampleBlockForColumns(required_columns, storage->getVirtuals(), storage->getStorageID())));
if (query_info.prewhere_info)
{
if (query_info.prewhere_info->alias_actions)
{
pipe.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ExpressionTransform>(header, query_info.prewhere_info->alias_actions);
});
}
pipe.addSimpleTransform([&](const Block & header)
{
return std::make_shared<FilterTransform>(
header,
query_info.prewhere_info->prewhere_actions,
query_info.prewhere_info->prewhere_column_name,
query_info.prewhere_info->remove_prewhere_column);
});
// To remove additional columns
// In some cases, we did not read any marks so that the pipeline.streams is empty
// Thus, some columns in prewhere are not removed as expected
// This leads to mismatched header in distributed table
if (query_info.prewhere_info->remove_columns_actions)
{
pipe.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ExpressionTransform>(
header, query_info.prewhere_info->remove_columns_actions);
});
}
}
}
pipeline = std::make_unique<QueryPipeline>();
QueryPipelineProcessorsCollector collector(*pipeline, this);
pipe.setLimits(limits);
/**
* Leaf size limits should be applied only for local processing of distributed queries.
* Such limits allow to control the read stage on leaf nodes and exclude the merging stage.
* Consider the case when distributed query needs to read from multiple shards. Then leaf
* limits will be applied on the shards only (including the root node) but will be ignored
* on the results merging stage.
*/
if (!storage->isRemote())
pipe.setLeafLimits(leaf_limits);
if (quota)
pipe.setQuota(quota);
pipeline->init(std::move(pipe));
/// Add resources to pipeline. The order is important.
/// Add in reverse order of destruction. Pipeline will be destroyed at the end in case of exception.
pipeline->addInterpreterContext(std::move(context));
pipeline->addStorageHolder(std::move(storage));
pipeline->addTableLock(std::move(table_lock));
processors = collector.detachProcessors();
output_stream = DataStream{.header = pipeline->getHeader(), .has_single_port = pipeline->getNumStreams() == 1};
}
ReadFromStorageStep::~ReadFromStorageStep() = default;
QueryPipelinePtr ReadFromStorageStep::updatePipeline(QueryPipelines)
{
return std::move(pipeline);
}
void ReadFromStorageStep::describePipeline(FormatSettings & settings) const
{
IQueryPlanStep::describePipeline(processors, settings);
}
}

53
src/Processors/QueryPlan/ReadFromStorageStep.h
View File

@ -1,53 +0,0 @@
#pragma once
#include <Processors/QueryPlan/IQueryPlanStep.h>
#include <Core/QueryProcessingStage.h>
#include <Storages/TableLockHolder.h>
#include <DataStreams/StreamLocalLimits.h>
namespace DB
{
class IStorage;
using StoragePtr = std::shared_ptr<IStorage>;
struct StorageInMemoryMetadata;
using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
struct SelectQueryInfo;
struct PrewhereInfo;
class EnabledQuota;
/// Reads from storage.
class ReadFromStorageStep : public IQueryPlanStep
{
public:
ReadFromStorageStep(
TableLockHolder table_lock,
StorageMetadataPtr metadata_snapshot,
StreamLocalLimits & limits,
SizeLimits & leaf_limits,
std::shared_ptr<const EnabledQuota> quota,
StoragePtr storage,
const Names & required_columns,
SelectQueryInfo & query_info,
std::shared_ptr<Context> context,
QueryProcessingStage::Enum processing_stage,
size_t max_block_size,
size_t max_streams);
~ReadFromStorageStep() override;
String getName() const override { return "ReadFromStorage"; }
QueryPipelinePtr updatePipeline(QueryPipelines) override;
void describePipeline(FormatSettings & settings) const override;
private:
QueryPipelinePtr pipeline;
Processors processors;
};
}

37
src/Processors/QueryPlan/ReverseRowsStep.cpp Normal file
View File

@ -0,0 +1,37 @@
#include <Processors/QueryPlan/ReverseRowsStep.h>
#include <Processors/QueryPipeline.h>
#include <Processors/Transforms/ReverseTransform.h>
namespace DB
{
static ITransformingStep::Traits getTraits()
{
return ITransformingStep::Traits
{
{
.preserves_distinct_columns = true,
.returns_single_stream = false,
.preserves_number_of_streams = true,
.preserves_sorting = false,
},
{
.preserves_number_of_rows = true,
}
};
}
ReverseRowsStep::ReverseRowsStep(const DataStream & input_stream_)
: ITransformingStep(input_stream_, input_stream_.header, getTraits())
{
}
void ReverseRowsStep::transformPipeline(QueryPipeline & pipeline)
{
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ReverseTransform>(header);
});
}
}

18
src/Processors/QueryPlan/ReverseRowsStep.h Normal file
View File

@ -0,0 +1,18 @@
#pragma once
#include <Processors/QueryPlan/ITransformingStep.h>
namespace DB
{
/// Reverse rows in chunk.
class ReverseRowsStep : public ITransformingStep
{
public:
ReverseRowsStep(const DataStream & input_stream_);
String getName() const override { return "ReverseRows"; }
void transformPipeline(QueryPipeline & pipeline) override;
};
}

71
src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp Normal file
View File

@ -0,0 +1,71 @@
#include <Processors/QueryPlan/SettingQuotaAndLimitsStep.h>
#include <Processors/QueryPipeline.h>
#include <Storages/IStorage.h>
namespace DB
{
static ITransformingStep::Traits getTraits()
{
return ITransformingStep::Traits
{
{
.preserves_distinct_columns = true,
.returns_single_stream = false,
.preserves_number_of_streams = true,
.preserves_sorting = true,
},
{
.preserves_number_of_rows = true,
}
};
}
SettingQuotaAndLimitsStep::SettingQuotaAndLimitsStep(
const DataStream & input_stream_,
StoragePtr storage_,
TableLockHolder table_lock_,
StreamLocalLimits & limits_,
SizeLimits & leaf_limits_,
std::shared_ptr<const EnabledQuota> quota_,
std::shared_ptr<Context> context_)
: ITransformingStep(input_stream_, input_stream_.header, getTraits())
, context(std::move(context_))
, storage(std::move(storage_))
, table_lock(std::move(table_lock_))
, limits(limits_)
, leaf_limits(leaf_limits_)
, quota(std::move(quota_))
{
}
void SettingQuotaAndLimitsStep::transformPipeline(QueryPipeline & pipeline)
{
/// Table lock is stored inside pipeline here.
pipeline.setLimits(limits);
/**
* Leaf size limits should be applied only for local processing of distributed queries.
* Such limits allow to control the read stage on leaf nodes and exclude the merging stage.
* Consider the case when distributed query needs to read from multiple shards. Then leaf
* limits will be applied on the shards only (including the root node) but will be ignored
* on the results merging stage.
*/
if (!storage->isRemote())
pipeline.setLeafLimits(leaf_limits);
if (quota)
pipeline.setQuota(quota);
/// Order of resources below is important.
if (context)
pipeline.addInterpreterContext(std::move(context));
if (storage)
pipeline.addStorageHolder(std::move(storage));
if (table_lock)
pipeline.addTableLock(std::move(table_lock));
}
}

44
src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h Normal file
View File

@ -0,0 +1,44 @@
#pragma once
#include <Processors/QueryPlan/ITransformingStep.h>
#include <Storages/TableLockHolder.h>
#include <DataStreams/StreamLocalLimits.h>
namespace DB
{
class IStorage;
using StoragePtr = std::shared_ptr<IStorage>;
struct StorageInMemoryMetadata;
using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
class EnabledQuota;
/// Add limits, quota, table_lock and other stuff to pipeline.
/// Doesn't change DataStream.
class SettingQuotaAndLimitsStep : public ITransformingStep
{
public:
SettingQuotaAndLimitsStep(
const DataStream & input_stream_,
StoragePtr storage_,
TableLockHolder table_lock_,
StreamLocalLimits & limits_,
SizeLimits & leaf_limits_,
std::shared_ptr<const EnabledQuota> quota_,
std::shared_ptr<Context> context_);
String getName() const override { return "SettingQuotaAndLimits"; }
void transformPipeline(QueryPipeline & pipeline) override;
private:
std::shared_ptr<Context> context;
StoragePtr storage;
TableLockHolder table_lock;
StreamLocalLimits limits;
SizeLimits leaf_limits;
std::shared_ptr<const EnabledQuota> quota;
};
}

2
src/Processors/QueryPlan/UnionStep.cpp
View File

@ -30,7 +30,7 @@ QueryPipelinePtr UnionStep::updatePipeline(QueryPipelines pipelines)
return pipeline;
}
*pipeline = QueryPipeline::unitePipelines(std::move(pipelines), output_stream->header ,max_threads);
*pipeline = QueryPipeline::unitePipelines(std::move(pipelines), output_stream->header, max_threads);
processors = collector.detachProcessors();
return pipeline;

2
src/Processors/QueryPlan/UnionStep.h
View File

@ -9,7 +9,7 @@ class UnionStep : public IQueryPlanStep
{
public:
/// max_threads is used to limit the number of threads for result pipeline.
UnionStep(DataStreams input_streams_, Block result_header, size_t max_threads_);
UnionStep(DataStreams input_streams_, Block result_header, size_t max_threads_ = 0);
String getName() const override { return "Union"; }

33
src/Processors/Transforms/AddingConstColumnTransform.h
View File

@ -4,33 +4,40 @@
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
/// Adds a materialized const column to the chunk with a specified value.
template <typename T>
class AddingConstColumnTransform : public ISimpleTransform
{
public:
AddingConstColumnTransform(const Block & header, DataTypePtr data_type_, T value_, const String & column_name_)
: ISimpleTransform(header, addColumn(header, data_type_, column_name_), false)
, data_type(std::move(data_type_)), value(value_) {}
AddingConstColumnTransform(const Block & header, ColumnWithTypeAndName column_)
: ISimpleTransform(header, transformHeader(header, column_), false)
, column(std::move(column_))
{
if (!column.column || !isColumnConst(*column.column) || !column.column->empty())
throw Exception("AddingConstColumnTransform expected empty const column", ErrorCodes::LOGICAL_ERROR);
}
String getName() const override { return "AddingConstColumnTransform"; }
static Block transformHeader(Block header, ColumnWithTypeAndName & column_)
{
header.insert(column_);
return header;
}
protected:
void transform(Chunk & chunk) override
{
auto num_rows = chunk.getNumRows();
chunk.addColumn(data_type->createColumnConst(num_rows, value)->convertToFullColumnIfConst());
chunk.addColumn(column.column->cloneResized(num_rows)->convertToFullColumnIfConst());
}
private:
static Block addColumn(Block header, const DataTypePtr & data_type, const String & column_name)
{
header.insert({data_type->createColumn(), data_type, column_name});
return header;
}
DataTypePtr data_type;
T value;
ColumnWithTypeAndName column;
};
}

7
src/Processors/ya.make
View File

@ -88,7 +88,9 @@ SRCS(
Pipe.cpp
Port.cpp
QueryPipeline.cpp
QueryPlan/AddingConstColumnStep.cpp
QueryPlan/AddingDelayedSourceStep.cpp
QueryPlan/AddingMissedStep.cpp
QueryPlan/AggregatingStep.cpp
QueryPlan/ArrayJoinStep.cpp
QueryPlan/ConvertingStep.cpp
@ -105,16 +107,19 @@ SRCS(
QueryPlan/ITransformingStep.cpp
QueryPlan/LimitByStep.cpp
QueryPlan/LimitStep.cpp
QueryPlan/MaterializingStep.cpp
QueryPlan/MergeSortingStep.cpp
QueryPlan/MergingAggregatedStep.cpp
QueryPlan/MergingFinal.cpp
QueryPlan/MergingSortedStep.cpp
QueryPlan/OffsetStep.cpp
QueryPlan/PartialSortingStep.cpp
QueryPlan/QueryPlan.cpp
QueryPlan/ReadFromPreparedSource.cpp
QueryPlan/ReadFromStorageStep.cpp
QueryPlan/ReadNothingStep.cpp
QueryPlan/ReverseRowsStep.cpp
QueryPlan/RollupStep.cpp
QueryPlan/SettingQuotaAndLimitsStep.cpp
QueryPlan/TotalsHavingStep.cpp
QueryPlan/UnionStep.cpp
ResizeProcessor.cpp

277
src/Server/HTTPHandler.cpp
View File

@ -5,39 +5,39 @@
#include <chrono>
#include <iomanip>
#include <Compression/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <Core/ExternalTable.h>
#include <DataStreams/IBlockInputStream.h>
#include <Disks/StoragePolicy.h>
#include <IO/CascadeWriteBuffer.h>
#include <IO/ConcatReadBuffer.h>
#include <IO/MemoryReadWriteBuffer.h>
#include <IO/ReadBufferFromIStream.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteBufferFromTemporaryFile.h>
#include <IO/WriteHelpers.h>
#include <IO/copyData.h>
#include <Interpreters/QueryParameterVisitor.h>
#include <Interpreters/executeQuery.h>
#include <Poco/File.h>
#include <Poco/Net/HTTPBasicCredentials.h>
#include <Poco/Net/HTTPRequestHandlerFactory.h>
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerRequestImpl.h>
#include <Poco/Net/HTTPServerResponse.h>
#include <Poco/Net/HTTPRequestHandlerFactory.h>
#include <Poco/Net/HTTPStream.h>
#include <Poco/Net/NetException.h>
#include <ext/scope_guard.h>
#include <Core/ExternalTable.h>
#include <Common/SettingsChanges.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/escapeForFileName.h>
#include <common/getFQDNOrHostName.h>
#include <Common/setThreadName.h>
#include <Common/SettingsChanges.h>
#include <Disks/StoragePolicy.h>
#include <Compression/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <IO/ReadBufferFromIStream.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteHelpers.h>
#include <IO/copyData.h>
#include <IO/ConcatReadBuffer.h>
#include <IO/CascadeWriteBuffer.h>
#include <IO/MemoryReadWriteBuffer.h>
#include <IO/WriteBufferFromTemporaryFile.h>
#include <DataStreams/IBlockInputStream.h>
#include <Interpreters/executeQuery.h>
#include <Interpreters/QueryParameterVisitor.h>
#include <Common/typeid_cast.h>
#include <Poco/Net/HTTPStream.h>
#include <common/getFQDNOrHostName.h>
#include <ext/scope_guard.h>
#if !defined(ARCADIA_BUILD)
# include <Common/config.h>
@ -46,10 +46,8 @@
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int CANNOT_PARSE_TEXT;
extern const int CANNOT_PARSE_ESCAPE_SEQUENCE;
@ -109,36 +107,25 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti
{
return HTTPResponse::HTTP_UNAUTHORIZED;
}
else if (exception_code == ErrorCodes::CANNOT_PARSE_TEXT ||
exception_code == ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE ||
exception_code == ErrorCodes::CANNOT_PARSE_QUOTED_STRING ||
exception_code == ErrorCodes::CANNOT_PARSE_DATE ||
exception_code == ErrorCodes::CANNOT_PARSE_DATETIME ||
exception_code == ErrorCodes::CANNOT_PARSE_NUMBER ||
exception_code == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED ||
exception_code == ErrorCodes::UNKNOWN_ELEMENT_IN_AST ||
exception_code == ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE ||
exception_code == ErrorCodes::TOO_DEEP_AST ||
exception_code == ErrorCodes::TOO_BIG_AST ||
exception_code == ErrorCodes::UNEXPECTED_AST_STRUCTURE ||
exception_code == ErrorCodes::SYNTAX_ERROR ||
exception_code == ErrorCodes::INCORRECT_DATA ||
exception_code == ErrorCodes::TYPE_MISMATCH)
else if (
exception_code == ErrorCodes::CANNOT_PARSE_TEXT || exception_code == ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE
|| exception_code == ErrorCodes::CANNOT_PARSE_QUOTED_STRING || exception_code == ErrorCodes::CANNOT_PARSE_DATE
|| exception_code == ErrorCodes::CANNOT_PARSE_DATETIME || exception_code == ErrorCodes::CANNOT_PARSE_NUMBER
|| exception_code == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED || exception_code == ErrorCodes::UNKNOWN_ELEMENT_IN_AST
|| exception_code == ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE || exception_code == ErrorCodes::TOO_DEEP_AST
|| exception_code == ErrorCodes::TOO_BIG_AST || exception_code == ErrorCodes::UNEXPECTED_AST_STRUCTURE
|| exception_code == ErrorCodes::SYNTAX_ERROR || exception_code == ErrorCodes::INCORRECT_DATA
|| exception_code == ErrorCodes::TYPE_MISMATCH)
{
return HTTPResponse::HTTP_BAD_REQUEST;
}
else if (exception_code == ErrorCodes::UNKNOWN_TABLE ||
exception_code == ErrorCodes::UNKNOWN_FUNCTION ||
exception_code == ErrorCodes::UNKNOWN_IDENTIFIER ||
exception_code == ErrorCodes::UNKNOWN_TYPE ||
exception_code == ErrorCodes::UNKNOWN_STORAGE ||
exception_code == ErrorCodes::UNKNOWN_DATABASE ||
exception_code == ErrorCodes::UNKNOWN_SETTING ||
exception_code == ErrorCodes::UNKNOWN_DIRECTION_OF_SORTING ||
exception_code == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION ||
exception_code == ErrorCodes::UNKNOWN_FORMAT ||
exception_code == ErrorCodes::UNKNOWN_DATABASE_ENGINE ||
exception_code == ErrorCodes::UNKNOWN_TYPE_OF_QUERY)
else if (
exception_code == ErrorCodes::UNKNOWN_TABLE || exception_code == ErrorCodes::UNKNOWN_FUNCTION
|| exception_code == ErrorCodes::UNKNOWN_IDENTIFIER || exception_code == ErrorCodes::UNKNOWN_TYPE
|| exception_code == ErrorCodes::UNKNOWN_STORAGE || exception_code == ErrorCodes::UNKNOWN_DATABASE
|| exception_code == ErrorCodes::UNKNOWN_SETTING || exception_code == ErrorCodes::UNKNOWN_DIRECTION_OF_SORTING
|| exception_code == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION || exception_code == ErrorCodes::UNKNOWN_FORMAT
|| exception_code == ErrorCodes::UNKNOWN_DATABASE_ENGINE || exception_code == ErrorCodes::UNKNOWN_TYPE_OF_QUERY)
{
return HTTPResponse::HTTP_NOT_FOUND;
}
@ -150,8 +137,7 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti
{
return HTTPResponse::HTTP_NOT_IMPLEMENTED;
}
else if (exception_code == ErrorCodes::SOCKET_TIMEOUT ||
exception_code == ErrorCodes::CANNOT_OPEN_FILE)
else if (exception_code == ErrorCodes::SOCKET_TIMEOUT || exception_code == ErrorCodes::CANNOT_OPEN_FILE)
{
return HTTPResponse::HTTP_SERVICE_UNAVAILABLE;
}
@ -164,9 +150,7 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti
}
static std::chrono::steady_clock::duration parseSessionTimeout(
const Poco::Util::AbstractConfiguration & config,
const HTMLForm & params)
static std::chrono::steady_clock::duration parseSessionTimeout(const Poco::Util::AbstractConfiguration & config, const HTMLForm & params)
{
unsigned session_timeout = config.getInt("default_session_timeout", 60);
@ -180,8 +164,9 @@ static std::chrono::steady_clock::duration parseSessionTimeout(
throw Exception("Invalid session timeout: '" + session_timeout_str + "'", ErrorCodes::INVALID_SESSION_TIMEOUT);
if (session_timeout > max_session_timeout)
throw Exception("Session timeout '" + session_timeout_str + "' is larger than max_session_timeout: " + toString(max_session_timeout)
+ ". Maximum session timeout could be modified in configuration file.",
throw Exception(
"Session timeout '" + session_timeout_str + "' is larger than max_session_timeout: " + toString(max_session_timeout)
+ ". Maximum session timeout could be modified in configuration file.",
ErrorCodes::INVALID_SESSION_TIMEOUT);
}
@ -209,8 +194,7 @@ void HTTPHandler::pushDelayedResults(Output & used_output)
IReadableWriteBuffer * write_buf_concrete;
ReadBufferPtr reread_buf;
if (write_buf
&& (write_buf_concrete = dynamic_cast<IReadableWriteBuffer *>(write_buf.get()))
if (write_buf && (write_buf_concrete = dynamic_cast<IReadableWriteBuffer *>(write_buf.get()))
&& (reread_buf = write_buf_concrete->tryGetReadBuffer()))
{
read_buffers.emplace_back(reread_buf);
@ -223,9 +207,7 @@ void HTTPHandler::pushDelayedResults(Output & used_output)
}
HTTPHandler::HTTPHandler(IServer & server_, const std::string & name)
: server(server_)
, log(&Poco::Logger::get(name))
HTTPHandler::HTTPHandler(IServer & server_, const std::string & name) : server(server_), log(&Poco::Logger::get(name))
{
server_display_name = server.config().getString("display_name", getFQDNOrHostName());
}
@ -271,12 +253,12 @@ void HTTPHandler::processQuery(
else
{
/// It is prohibited to mix different authorization schemes.
if (request.hasCredentials()
|| params.has("user")
|| params.has("password")
|| params.has("quota_key"))
if (request.hasCredentials() || params.has("user") || params.has("password") || params.has("quota_key"))
{
throw Exception("Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously", ErrorCodes::REQUIRED_PASSWORD);
throw Exception(
"Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods "
"simultaneously",
ErrorCodes::REQUIRED_PASSWORD);
}
}
@ -318,12 +300,13 @@ void HTTPHandler::processQuery(
{
std::string opentelemetry_traceparent = request.get("traceparent");
std::string error;
if (!context.getClientInfo().parseTraceparentHeader(
opentelemetry_traceparent, error))
if (!context.getClientInfo().parseTraceparentHeader(opentelemetry_traceparent, error))
{
throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER,
throw Exception(
ErrorCodes::BAD_REQUEST_PARAMETER,
"Failed to parse OpenTelemetry traceparent header '{}': {}",
opentelemetry_traceparent, error);
opentelemetry_traceparent,
error);
}
context.getClientInfo().opentelemetry_tracestate = request.get("tracestate", "");
@ -332,8 +315,7 @@ void HTTPHandler::processQuery(
// Set the query id supplied by the user, if any, and also update the
// OpenTelemetry fields.
context.setCurrentQueryId(params.get("query_id",
request.get("X-ClickHouse-Query-Id", "")));
context.setCurrentQueryId(params.get("query_id", request.get("X-ClickHouse-Query-Id", "")));
/// The client can pass a HTTP header indicating supported compression method (gzip or deflate).
String http_response_compression_methods = request.get("Accept-Encoding", "");
@ -351,6 +333,8 @@ void HTTPHandler::processQuery(
http_response_compression_method = CompressionMethod::Gzip;
else if (std::string::npos != http_response_compression_methods.find("deflate"))
http_response_compression_method = CompressionMethod::Zlib;
else if (std::string::npos != http_response_compression_methods.find("xz"))
http_response_compression_method = CompressionMethod::Xz;
}
bool client_supports_http_compression = http_response_compression_method != CompressionMethod::None;
@ -360,8 +344,8 @@ void HTTPHandler::processQuery(
bool internal_compression = params.getParsed<bool>("compress", false);
/// At least, we should postpone sending of first buffer_size result bytes
size_t buffer_size_total = std::max(
params.getParsed<size_t>("buffer_size", DBMS_DEFAULT_BUFFER_SIZE), static_cast<size_t>(DBMS_DEFAULT_BUFFER_SIZE));
size_t buffer_size_total
= std::max(params.getParsed<size_t>("buffer_size", DBMS_DEFAULT_BUFFER_SIZE), static_cast<size_t>(DBMS_DEFAULT_BUFFER_SIZE));
/// If it is specified, the whole result will be buffered.
/// First ~buffer_size bytes will be buffered in memory, the remaining bytes will be stored in temporary file.
@ -393,23 +377,20 @@ void HTTPHandler::processQuery(
const std::string tmp_path(context.getTemporaryVolume()->getDisk()->getPath());
const std::string tmp_path_template(tmp_path + "http_buffers/");
auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &)
{
return WriteBufferFromTemporaryFile::create(tmp_path_template);
};
auto create_tmp_disk_buffer
= [tmp_path_template](const WriteBufferPtr &) { return WriteBufferFromTemporaryFile::create(tmp_path_template); };
cascade_buffer2.emplace_back(std::move(create_tmp_disk_buffer));
}
else
{
auto push_memory_buffer_and_continue = [next_buffer = used_output.out_maybe_compressed] (const WriteBufferPtr & prev_buf)
{
auto push_memory_buffer_and_continue = [next_buffer = used_output.out_maybe_compressed](const WriteBufferPtr & prev_buf) {
auto * prev_memory_buffer = typeid_cast<MemoryWriteBuffer *>(prev_buf.get());
if (!prev_memory_buffer)
throw Exception("Expected MemoryWriteBuffer", ErrorCodes::LOGICAL_ERROR);
auto rdbuf = prev_memory_buffer->tryGetReadBuffer();
copyData(*rdbuf , *next_buffer);
copyData(*rdbuf, *next_buffer);
return next_buffer;
};
@ -417,8 +398,8 @@ void HTTPHandler::processQuery(
cascade_buffer2.emplace_back(push_memory_buffer_and_continue);
}
used_output.out_maybe_delayed_and_compressed = std::make_shared<CascadeWriteBuffer>(
std::move(cascade_buffer1), std::move(cascade_buffer2));
used_output.out_maybe_delayed_and_compressed
= std::make_shared<CascadeWriteBuffer>(std::move(cascade_buffer1), std::move(cascade_buffer2));
}
else
{
@ -444,13 +425,23 @@ void HTTPHandler::processQuery(
std::unique_ptr<ReadBuffer> in;
static const NameSet reserved_param_names{"compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace",
"buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check"};
static const NameSet reserved_param_names{
"compress",
"decompress",
"user",
"password",
"quota_key",
"query_id",
"stacktrace",
"buffer_size",
"wait_end_of_query",
"session_id",
"session_timeout",
"session_check"};
Names reserved_param_suffixes;
auto param_could_be_skipped = [&] (const String & name)
{
auto param_could_be_skipped = [&](const String & name) {
/// Empty parameter appears when URL like ?&a=b or a=b&&c=d. Just skip them for user's convenience.
if (name.empty())
return true;
@ -575,12 +566,10 @@ void HTTPHandler::processQuery(
client_info.http_method = http_method;
client_info.http_user_agent = request.get("User-Agent", "");
auto append_callback = [&context] (ProgressCallback callback)
{
auto append_callback = [&context](ProgressCallback callback) {
auto prev = context.getProgressCallback();
context.setProgressCallback([prev, callback] (const Progress & progress)
{
context.setProgressCallback([prev, callback](const Progress & progress) {
if (prev)
prev(progress);
@ -590,14 +579,13 @@ void HTTPHandler::processQuery(
/// While still no data has been sent, we will report about query execution progress by sending HTTP headers.
if (settings.send_progress_in_http_headers)
append_callback([&used_output] (const Progress & progress) { used_output.out->onProgress(progress); });
append_callback([&used_output](const Progress & progress) { used_output.out->onProgress(progress); });
if (settings.readonly > 0 && settings.cancel_http_readonly_queries_on_client_close)
{
Poco::Net::StreamSocket & socket = dynamic_cast<Poco::Net::HTTPServerRequestImpl &>(request).socket();
append_callback([&context, &socket](const Progress &)
{
append_callback([&context, &socket](const Progress &) {
/// Assume that at the point this method is called no one is reading data from the socket any more.
/// True for read-only queries.
try
@ -621,15 +609,17 @@ void HTTPHandler::processQuery(
query_scope.emplace(context);
executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context,
[&response] (const String & current_query_id, const String & content_type, const String & format, const String & timezone)
{
executeQuery(
*in,
*used_output.out_maybe_delayed_and_compressed,
/* allow_into_outfile = */ false,
context,
[&response](const String & current_query_id, const String & content_type, const String & format, const String & timezone) {
response.setContentType(content_type);
response.add("X-ClickHouse-Query-Id", current_query_id);
response.add("X-ClickHouse-Format", format);
response.add("X-ClickHouse-Timezone", timezone);
}
);
});
if (used_output.hasDelayed())
{
@ -642,8 +632,11 @@ void HTTPHandler::processQuery(
used_output.out->finalize();
}
void HTTPHandler::trySendExceptionToClient(const std::string & s, int exception_code,
Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response,
void HTTPHandler::trySendExceptionToClient(
const std::string & s,
int exception_code,
Poco::Net::HTTPServerRequest & request,
Poco::Net::HTTPServerResponse & response,
Output & used_output)
{
try
@ -652,17 +645,14 @@ void HTTPHandler::trySendExceptionToClient(const std::string & s, int exception_
/// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body
/// to avoid reading part of the current request body in the next request.
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST
&& response.getKeepAlive()
&& !request.stream().eof()
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && response.getKeepAlive() && !request.stream().eof()
&& exception_code != ErrorCodes::HTTP_LENGTH_REQUIRED)
{
request.stream().ignore(std::numeric_limits<std::streamsize>::max());
}
bool auth_fail = exception_code == ErrorCodes::UNKNOWN_USER ||
exception_code == ErrorCodes::WRONG_PASSWORD ||
exception_code == ErrorCodes::REQUIRED_PASSWORD;
bool auth_fail = exception_code == ErrorCodes::UNKNOWN_USER || exception_code == ErrorCodes::WRONG_PASSWORD
|| exception_code == ErrorCodes::REQUIRED_PASSWORD;
if (auth_fail)
{
@ -740,10 +730,12 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
with_stacktrace = params.getParsed<bool>("stacktrace", false);
/// Workaround. Poco does not detect 411 Length Required case.
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding() &&
!request.hasContentLength())
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding()
&& !request.hasContentLength())
{
throw Exception("The Transfer-Encoding is not chunked and there is no Content-Length header for POST request", ErrorCodes::HTTP_LENGTH_REQUIRED);
throw Exception(
"The Transfer-Encoding is not chunked and there is no Content-Length header for POST request",
ErrorCodes::HTTP_LENGTH_REQUIRED);
}
processQuery(context, request, params, response, used_output, query_scope);
@ -771,7 +763,7 @@ DynamicQueryHandler::DynamicQueryHandler(IServer & server_, const std::string &
bool DynamicQueryHandler::customizeQueryParam(Context & context, const std::string & key, const std::string & value)
{
if (key == param_name)
return true; /// do nothing
return true; /// do nothing
if (startsWith(key, "param_"))
{
@ -786,7 +778,6 @@ bool DynamicQueryHandler::customizeQueryParam(Context & context, const std::stri
std::string DynamicQueryHandler::getQuery(Poco::Net::HTTPServerRequest & request, HTMLForm & params, Context & context)
{
if (likely(!startsWith(request.getContentType(), "multipart/form-data")))
{
/// Part of the query can be passed in the 'query' parameter and the rest in the request body
@ -811,10 +802,16 @@ std::string DynamicQueryHandler::getQuery(Poco::Net::HTTPServerRequest & request
}
PredefinedQueryHandler::PredefinedQueryHandler(
IServer & server_, const NameSet & receive_params_, const std::string & predefined_query_
, const CompiledRegexPtr & url_regex_, const std::unordered_map<String, CompiledRegexPtr> & header_name_with_regex_)
: HTTPHandler(server_, "PredefinedQueryHandler"), receive_params(receive_params_), predefined_query(predefined_query_)
, url_regex(url_regex_), header_name_with_capture_regex(header_name_with_regex_)
IServer & server_,
const NameSet & receive_params_,
const std::string & predefined_query_,
const CompiledRegexPtr & url_regex_,
const std::unordered_map<String, CompiledRegexPtr> & header_name_with_regex_)
: HTTPHandler(server_, "PredefinedQueryHandler")
, receive_params(receive_params_)
, predefined_query(predefined_query_)
, url_regex(url_regex_)
, header_name_with_capture_regex(header_name_with_regex_)
{
}
@ -834,8 +831,7 @@ void PredefinedQueryHandler::customizeContext(Poco::Net::HTTPServerRequest & req
/// If in the configuration file, the handler's header is regex and contains named capture group
/// We will extract regex named capture groups as query parameters
const auto & set_query_params = [&](const char * begin, const char * end, const CompiledRegexPtr & compiled_regex)
{
const auto & set_query_params = [&](const char * begin, const char * end, const CompiledRegexPtr & compiled_regex) {
int num_captures = compiled_regex->NumberOfCapturingGroups() + 1;
re2::StringPiece matches[num_captures];
@ -880,16 +876,16 @@ std::string PredefinedQueryHandler::getQuery(Poco::Net::HTTPServerRequest & requ
Poco::Net::HTTPRequestHandlerFactory * createDynamicHandlerFactory(IServer & server, const std::string & config_prefix)
{
std::string query_param_name = server.config().getString(config_prefix + ".handler.query_param_name", "query");
return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory<DynamicQueryHandler>(server, std::move(query_param_name)), server.config(), config_prefix);
return addFiltersFromConfig(
new HandlingRuleHTTPHandlerFactory<DynamicQueryHandler>(server, std::move(query_param_name)), server.config(), config_prefix);
}
static inline bool capturingNamedQueryParam(NameSet receive_params, const CompiledRegexPtr & compiled_regex)
{
const auto & capturing_names = compiled_regex->NamedCapturingGroups();
return std::count_if(capturing_names.begin(), capturing_names.end(), [&](const auto & iterator)
{
return std::count_if(receive_params.begin(), receive_params.end(),
[&](const auto & param_name) { return param_name == iterator.first; });
return std::count_if(capturing_names.begin(), capturing_names.end(), [&](const auto & iterator) {
return std::count_if(
receive_params.begin(), receive_params.end(), [&](const auto & param_name) { return param_name == iterator.first; });
});
}
@ -898,8 +894,10 @@ static inline CompiledRegexPtr getCompiledRegex(const std::string & expression)
auto compiled_regex = std::make_shared<const re2::RE2>(expression);
if (!compiled_regex->ok())
throw Exception("Cannot compile re2: " + expression + " for http handling rule, error: " +
compiled_regex->error() + ". Look at https://github.com/google/re2/wiki/Syntax for reference.", ErrorCodes::CANNOT_COMPILE_REGEXP);
throw Exception(
"Cannot compile re2: " + expression + " for http handling rule, error: " + compiled_regex->error()
+ ". Look at https://github.com/google/re2/wiki/Syntax for reference.",
ErrorCodes::CANNOT_COMPILE_REGEXP);
return compiled_regex;
}
@ -909,7 +907,8 @@ Poco::Net::HTTPRequestHandlerFactory * createPredefinedHandlerFactory(IServer &
Poco::Util::AbstractConfiguration & configuration = server.config();
if (!configuration.has(config_prefix + ".handler.query"))
throw Exception("There is no path '" + config_prefix + ".handler.query" + "' in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
throw Exception(
"There is no path '" + config_prefix + ".handler.query" + "' in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
std::string predefined_query = configuration.getString(config_prefix + ".handler.query");
NameSet analyze_receive_params = analyzeReceiveQueryParams(predefined_query);
@ -940,14 +939,22 @@ Poco::Net::HTTPRequestHandlerFactory * createPredefinedHandlerFactory(IServer &
auto regex = getCompiledRegex(url_expression);
if (capturingNamedQueryParam(analyze_receive_params, regex))
return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>(
server, std::move(analyze_receive_params), std::move(predefined_query), std::move(regex),
std::move(headers_name_with_regex)), configuration, config_prefix);
return addFiltersFromConfig(
new HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>(
server,
std::move(analyze_receive_params),
std::move(predefined_query),
std::move(regex),
std::move(headers_name_with_regex)),
configuration,
config_prefix);
}
return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>(
server, std::move(analyze_receive_params), std::move(predefined_query), CompiledRegexPtr{} ,std::move(headers_name_with_regex)),
configuration, config_prefix);
return addFiltersFromConfig(
new HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>(
server, std::move(analyze_receive_params), std::move(predefined_query), CompiledRegexPtr{}, std::move(headers_name_with_regex)),
configuration,
config_prefix);
}
}

28
src/Storages/IStorage.cpp
View File

@ -7,11 +7,12 @@
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Processors/Pipe.h>
#include <Processors/QueryPlan/ReadFromStorageStep.h>
#include <Processors/QueryPlan/ReadFromPreparedSource.h>
#include <Interpreters/Context.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/quoteString.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/InterpreterSelectQuery.h>
namespace DB
@ -94,24 +95,25 @@ Pipe IStorage::read(
void IStorage::read(
QueryPlan & query_plan,
TableLockHolder table_lock,
StorageMetadataPtr metadata_snapshot,
StreamLocalLimits & limits,
SizeLimits & leaf_limits,
std::shared_ptr<const EnabledQuota> quota,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
std::shared_ptr<Context> context,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams)
{
auto read_step = std::make_unique<ReadFromStorageStep>(
std::move(table_lock), std::move(metadata_snapshot), limits, leaf_limits, std::move(quota), shared_from_this(),
column_names, query_info, std::move(context), processed_stage, max_block_size, num_streams);
read_step->setStepDescription("Read from " + getName());
query_plan.addStep(std::move(read_step));
auto pipe = read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
if (pipe.empty())
{
auto header = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID());
InterpreterSelectQuery::addEmptySourceToQueryPlan(query_plan, header, query_info);
}
else
{
auto read_step = std::make_unique<ReadFromStorageStep>(std::move(pipe), getName());
query_plan.addStep(std::move(read_step));
}
}
Pipe IStorage::alterPartition(

19
src/Storages/IStorage.h
View File

@ -48,6 +48,7 @@ using Processors = std::vector<ProcessorPtr>;
class Pipe;
class QueryPlan;
using QueryPlanPtr = std::unique_ptr<QueryPlan>;
class StoragePolicy;
using StoragePolicyPtr = std::shared_ptr<const StoragePolicy>;
@ -285,17 +286,13 @@ public:
/// Default implementation creates ReadFromStorageStep and uses usual read.
virtual void read(
QueryPlan & query_plan,
TableLockHolder table_lock,
StorageMetadataPtr metadata_snapshot,
StreamLocalLimits & limits,
SizeLimits & leaf_limits,
std::shared_ptr<const EnabledQuota> quota,
const Names & column_names,
SelectQueryInfo & query_info,
std::shared_ptr<Context> context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams);
const Names & /*column_names*/,
const StorageMetadataPtr & /*metadata_snapshot*/,
SelectQueryInfo & /*query_info*/,
const Context & /*context*/,
QueryProcessingStage::Enum /*processed_stage*/,
size_t /*max_block_size*/,
unsigned /*num_streams*/);
/** Writes the data to a table.
* Receives a description of the query, which can contain information about the data write method.

12
src/Storages/MergeTree/BoolMask.h
View File

@ -9,17 +9,17 @@ struct BoolMask
BoolMask() {}
BoolMask(bool can_be_true_, bool can_be_false_) : can_be_true(can_be_true_), can_be_false(can_be_false_) {}
BoolMask operator &(const BoolMask & m)
BoolMask operator &(const BoolMask & m) const
{
return BoolMask(can_be_true && m.can_be_true, can_be_false || m.can_be_false);
return {can_be_true && m.can_be_true, can_be_false || m.can_be_false};
}
BoolMask operator |(const BoolMask & m)
BoolMask operator |(const BoolMask & m) const
{
return BoolMask(can_be_true || m.can_be_true, can_be_false && m.can_be_false);
return {can_be_true || m.can_be_true, can_be_false && m.can_be_false};
}
BoolMask operator !()
BoolMask operator !() const
{
return BoolMask(can_be_false, can_be_true);
return {can_be_false, can_be_true};
}
/// If mask is (true, true), then it can no longer change under operation |.

14
src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp
View File

@ -1,6 +1,7 @@
#include <Storages/MergeTree/EphemeralLockInZooKeeper.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <common/logger_useful.h>
#include <common/types.h>
namespace DB
@ -71,13 +72,13 @@ EphemeralLockInZooKeeper::~EphemeralLockInZooKeeper()
EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions(
const String & block_numbers_path, const String & path_prefix, const String & temp_path,
zkutil::ZooKeeper & zookeeper_)
: zookeeper(zookeeper_)
: zookeeper(&zookeeper_)
{
std::vector<String> holders;
while (true)
{
Coordination::Stat partitions_stat;
Strings partitions = zookeeper.getChildren(block_numbers_path, &partitions_stat);
Strings partitions = zookeeper->getChildren(block_numbers_path, &partitions_stat);
if (holders.size() < partitions.size())
{
@ -85,7 +86,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions(
for (size_t i = 0; i < partitions.size() - holders.size(); ++i)
{
String path = temp_path + "/abandonable_lock-";
holder_futures.push_back(zookeeper.asyncCreate(path, {}, zkutil::CreateMode::EphemeralSequential));
holder_futures.push_back(zookeeper->asyncCreate(path, {}, zkutil::CreateMode::EphemeralSequential));
}
for (auto & future : holder_futures)
{
@ -104,7 +105,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions(
lock_ops.push_back(zkutil::makeCheckRequest(block_numbers_path, partitions_stat.version));
Coordination::Responses lock_responses;
Coordination::Error rc = zookeeper.tryMulti(lock_ops, lock_responses);
Coordination::Error rc = zookeeper->tryMulti(lock_ops, lock_responses);
if (rc == Coordination::Error::ZBADVERSION)
{
LOG_TRACE(&Poco::Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry.");
@ -131,13 +132,16 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions(
void EphemeralLocksInAllPartitions::unlock()
{
if (!zookeeper)
return;
std::vector<zkutil::ZooKeeper::FutureMulti> futures;
for (const auto & lock : locks)
{
Coordination::Requests unlock_ops;
unlock_ops.emplace_back(zkutil::makeRemoveRequest(lock.path, -1));
unlock_ops.emplace_back(zkutil::makeRemoveRequest(lock.holder_path, -1));
futures.push_back(zookeeper.asyncMulti(unlock_ops));
futures.push_back(zookeeper->asyncMulti(unlock_ops));
}
for (auto & future : futures)

69
src/Storages/MergeTree/EphemeralLockInZooKeeper.h
View File

@ -1,9 +1,14 @@
#pragma once
#include "ReplicatedMergeTreeMutationEntry.h"
#include <Common/ZooKeeper/ZooKeeper.h>
#include <Common/Exception.h>
#include <IO/ReadHelpers.h>
#include <map>
#include <optional>
namespace DB
{
@ -87,13 +92,30 @@ private:
/// Acquires block number locks in all partitions.
class EphemeralLocksInAllPartitions : private boost::noncopyable
class EphemeralLocksInAllPartitions : public boost::noncopyable
{
public:
EphemeralLocksInAllPartitions(
const String & block_numbers_path, const String & path_prefix, const String & temp_path,
zkutil::ZooKeeper & zookeeper_);
EphemeralLocksInAllPartitions() = default;
EphemeralLocksInAllPartitions(EphemeralLocksInAllPartitions && rhs) noexcept
: zookeeper(rhs.zookeeper)
, locks(std::move(rhs.locks))
{
rhs.zookeeper = nullptr;
}
EphemeralLocksInAllPartitions & operator=(EphemeralLocksInAllPartitions && rhs) noexcept
{
zookeeper = rhs.zookeeper;
rhs.zookeeper = nullptr;
locks = std::move(rhs.locks);
return *this;
}
struct LockInfo
{
String path;
@ -110,8 +132,51 @@ public:
~EphemeralLocksInAllPartitions();
private:
zkutil::ZooKeeper & zookeeper;
zkutil::ZooKeeper * zookeeper = nullptr;
std::vector<LockInfo> locks;
};
/// This class allows scoped manipulations with block numbers locked in certain partitions
/// See StorageReplicatedMergeTree::allocateBlockNumbersInAffectedPartitions and alter()/mutate() methods
class PartitionBlockNumbersHolder
{
public:
PartitionBlockNumbersHolder(const PartitionBlockNumbersHolder &) = delete;
PartitionBlockNumbersHolder & operator=(const PartitionBlockNumbersHolder &) = delete;
using BlockNumbersType = ReplicatedMergeTreeMutationEntry::BlockNumbersType;
PartitionBlockNumbersHolder() = default;
PartitionBlockNumbersHolder(
BlockNumbersType block_numbers_, std::optional<EphemeralLocksInAllPartitions> locked_block_numbers_holder)
: block_numbers(std::move(block_numbers_))
, multiple_partitions_holder(std::move(locked_block_numbers_holder))
{
}
PartitionBlockNumbersHolder(
BlockNumbersType block_numbers_, std::optional<EphemeralLockInZooKeeper> locked_block_numbers_holder)
: block_numbers(std::move(block_numbers_))
, single_partition_holder(std::move(locked_block_numbers_holder))
{
}
PartitionBlockNumbersHolder & operator=(PartitionBlockNumbersHolder &&) = default;
const BlockNumbersType & getBlockNumbers() const { return block_numbers; }
void reset()
{
multiple_partitions_holder.reset();
single_partition_holder.reset();
block_numbers.clear();
}
private:
BlockNumbersType block_numbers;
std::optional<EphemeralLocksInAllPartitions> multiple_partitions_holder;
std::optional<EphemeralLockInZooKeeper> single_partition_holder;
};
}

353
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -23,24 +23,20 @@
#include <Parsers/parseIdentifierOrStringLiteral.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Interpreters/Context.h>
#include <Processors/ConcatProcessor.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Processors/QueryPlan/FilterStep.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/ReadFromPreparedSource.h>
#include <Processors/QueryPlan/AddingConstColumnStep.h>
#include <Processors/QueryPlan/ReverseRowsStep.h>
#include <Processors/QueryPlan/MergingSortedStep.h>
#include <Processors/QueryPlan/UnionStep.h>
#include <Processors/QueryPlan/MergingFinal.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeEnum.h>
#include <DataTypes/DataTypesNumber.h>
#include <Processors/ConcatProcessor.h>
#include <Processors/Merges/AggregatingSortedTransform.h>
#include <Processors/Merges/CollapsingSortedTransform.h>
#include <Processors/Merges/MergingSortedTransform.h>
#include <Processors/Merges/ReplacingSortedTransform.h>
#include <Processors/Merges/SummingSortedTransform.h>
#include <Processors/Merges/VersionedCollapsingTransform.h>
#include <Processors/Sources/SourceFromInputStream.h>
#include <Processors/Transforms/AddingConstColumnTransform.h>
#include <Processors/Transforms/AddingSelectorTransform.h>
#include <Processors/Transforms/CopyTransform.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <Processors/Transforms/FilterTransform.h>
#include <Processors/Transforms/ReverseTransform.h>
#include <Storages/VirtualColumnUtils.h>
namespace ProfileEvents
@ -83,17 +79,6 @@ static Block getBlockWithPartColumn(const MergeTreeData::DataPartsVector & parts
return Block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "_part")};
}
/// Check if ORDER BY clause of the query has some expression.
static bool sortingDescriptionHasExpressions(const SortDescription & sort_description, const StorageMetadataPtr & metadata_snapshot)
{
auto all_columns = metadata_snapshot->getColumns();
for (const auto & sort_column : sort_description)
{
if (!all_columns.has(sort_column.column_name))
return true;
}
return false;
}
size_t MergeTreeDataSelectExecutor::getApproximateTotalRowsToRead(
const MergeTreeData::DataPartsVector & parts,
@ -144,7 +129,7 @@ static RelativeSize convertAbsoluteSampleSizeToRelative(const ASTPtr & node, siz
}
Pipe MergeTreeDataSelectExecutor::read(
QueryPlanPtr MergeTreeDataSelectExecutor::read(
const Names & column_names_to_return,
const StorageMetadataPtr & metadata_snapshot,
const SelectQueryInfo & query_info,
@ -159,7 +144,7 @@ Pipe MergeTreeDataSelectExecutor::read(
max_block_numbers_to_read);
}
Pipe MergeTreeDataSelectExecutor::readFromParts(
QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
MergeTreeData::DataPartsVector parts,
const Names & column_names_to_return,
const StorageMetadataPtr & metadata_snapshot,
@ -555,7 +540,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts(
if (no_data)
{
LOG_DEBUG(log, "Sampling yields no data.");
return {};
return std::make_unique<QueryPlan>();
}
LOG_DEBUG(log, "Key condition: {}", key_condition.toString());
@ -725,13 +710,13 @@ Pipe MergeTreeDataSelectExecutor::readFromParts(
LOG_DEBUG(log, "Selected {} parts by partition key, {} parts by primary key, {} marks by primary key, {} marks to read from {} ranges", parts.size(), parts_with_ranges.size(), sum_marks_pk.load(std::memory_order_relaxed), sum_marks, sum_ranges);
if (parts_with_ranges.empty())
return {};
return std::make_unique<QueryPlan>();
ProfileEvents::increment(ProfileEvents::SelectedParts, parts_with_ranges.size());
ProfileEvents::increment(ProfileEvents::SelectedRanges, sum_ranges);
ProfileEvents::increment(ProfileEvents::SelectedMarks, sum_marks);
Pipe res;
QueryPlanPtr plan;
/// Projection, that needed to drop columns, which have appeared by execution
/// of some extra expressions, and to allow execute the same expressions later.
@ -752,7 +737,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts(
std::sort(column_names_to_read.begin(), column_names_to_read.end());
column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end());
res = spreadMarkRangesAmongStreamsFinal(
plan = spreadMarkRangesAmongStreamsFinal(
std::move(parts_with_ranges),
num_streams,
column_names_to_read,
@ -774,7 +759,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts(
auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical());
auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActions(false);
res = spreadMarkRangesAmongStreamsWithOrder(
plan = spreadMarkRangesAmongStreamsWithOrder(
std::move(parts_with_ranges),
num_streams,
column_names_to_read,
@ -790,7 +775,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts(
}
else
{
res = spreadMarkRangesAmongStreams(
plan = spreadMarkRangesAmongStreams(
std::move(parts_with_ranges),
num_streams,
column_names_to_read,
@ -803,42 +788,52 @@ Pipe MergeTreeDataSelectExecutor::readFromParts(
reader_settings);
}
if (!plan)
return std::make_unique<QueryPlan>();
if (use_sampling)
{
res.addSimpleTransform([&filter_expression, &filter_function](const Block & header)
{
return std::make_shared<FilterTransform>(
header, filter_expression, filter_function->getColumnName(), false);
});
auto sampling_step = std::make_unique<FilterStep>(
plan->getCurrentDataStream(),
filter_expression,
filter_function->getColumnName(),
false);
sampling_step->setStepDescription("Sampling");
plan->addStep(std::move(sampling_step));
}
if (result_projection)
{
res.addSimpleTransform([&result_projection](const Block & header)
{
return std::make_shared<ExpressionTransform>(header, result_projection);
});
auto projection_step = std::make_unique<ExpressionStep>(plan->getCurrentDataStream(), result_projection);
projection_step->setStepDescription("Remove unused columns after reading from storage");
plan->addStep(std::move(projection_step));
}
/// By the way, if a distributed query or query to a Merge table is made, then the `_sample_factor` column can have different values.
if (sample_factor_column_queried)
{
res.addSimpleTransform([used_sample_factor](const Block & header)
{
return std::make_shared<AddingConstColumnTransform<Float64>>(
header, std::make_shared<DataTypeFloat64>(), used_sample_factor, "_sample_factor");
});
ColumnWithTypeAndName column;
column.name = "_sample_factor";
column.type = std::make_shared<DataTypeFloat64>();
column.column = column.type->createColumnConst(0, Field(used_sample_factor));
auto adding_column = std::make_unique<AddingConstColumnStep>(plan->getCurrentDataStream(), std::move(column));
adding_column->setStepDescription("Add _sample_factor column");
plan->addStep(std::move(adding_column));
}
if (query_info.prewhere_info && query_info.prewhere_info->remove_columns_actions)
{
res.addSimpleTransform([&query_info](const Block & header)
{
return std::make_shared<ExpressionTransform>(header, query_info.prewhere_info->remove_columns_actions);
});
auto expression_step = std::make_unique<ExpressionStep>(
plan->getCurrentDataStream(),
query_info.prewhere_info->remove_columns_actions);
expression_step->setStepDescription("Remove unused columns after PREWHERE");
plan->addStep(std::move(expression_step));
}
return res;
return plan;
}
namespace
@ -863,8 +858,20 @@ size_t roundRowsOrBytesToMarks(
}
static QueryPlanPtr createPlanFromPipe(Pipe pipe, const std::string & description = "")
{
auto plan = std::make_unique<QueryPlan>();
Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams(
std::string storage_name = "MergeTree";
if (!description.empty())
storage_name += ' ' + description;
auto step = std::make_unique<ReadFromStorageStep>(std::move(pipe), storage_name);
plan->addStep(std::move(step));
return plan;
}
QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams(
RangesInDataParts && parts,
size_t num_streams,
const Names & column_names,
@ -958,7 +965,7 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams(
res.emplace_back(std::move(source));
}
return Pipe::unitePipes(std::move(res));
return createPlanFromPipe(Pipe::unitePipes(std::move(res)));
}
else
{
@ -982,19 +989,18 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams(
if (pipe.numOutputPorts() > 1)
pipe.addTransform(std::make_shared<ConcatProcessor>(pipe.getHeader(), pipe.numOutputPorts()));
return pipe;
return createPlanFromPipe(std::move(pipe));
}
}
static ExpressionActionsPtr createProjection(const Pipe & pipe, const MergeTreeData & data)
static ExpressionActionsPtr createProjection(const Block & header, const MergeTreeData & data)
{
const auto & header = pipe.getHeader();
auto projection = std::make_shared<ExpressionActions>(header.getNamesAndTypesList(), data.global_context);
projection->add(ExpressionAction::project(header.getNames()));
return projection;
}
Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
RangesInDataParts && parts,
size_t num_streams,
const Names & column_names,
@ -1095,7 +1101,8 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
const size_t min_marks_per_stream = (sum_marks - 1) / num_streams + 1;
bool need_preliminary_merge = (parts.size() > settings.read_in_order_two_level_merge_threshold);
size_t max_output_ports = 0;
std::vector<QueryPlanPtr> plans;
for (size_t i = 0; i < num_streams && !parts.empty(); ++i)
{
@ -1195,60 +1202,64 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
}
}
auto pipe = Pipe::unitePipes(std::move(pipes));
auto plan = createPlanFromPipe(Pipe::unitePipes(std::move(pipes)), " with order");
if (input_order_info->direction != 1)
{
pipe.addSimpleTransform([](const Block & header)
{
return std::make_shared<ReverseTransform>(header);
});
auto reverse_step = std::make_unique<ReverseRowsStep>(plan->getCurrentDataStream());
plan->addStep(std::move(reverse_step));
}
max_output_ports = std::max(pipe.numOutputPorts(), max_output_ports);
res.emplace_back(std::move(pipe));
plans.emplace_back(std::move(plan));
}
if (need_preliminary_merge)
{
/// If ORDER BY clause of the query contains some expression,
/// then those new columns should be added for the merge step,
/// and this should be done always, if there is at least one pipe that
/// has multiple output ports.
bool sorting_key_has_expression = sortingDescriptionHasExpressions(input_order_info->order_key_prefix_descr, metadata_snapshot);
bool force_sorting_key_transform = res.size() > 1 && max_output_ports > 1 && sorting_key_has_expression;
SortDescription sort_description;
for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j)
sort_description.emplace_back(metadata_snapshot->getSortingKey().column_names[j],
input_order_info->direction, 1);
for (auto & pipe : res)
for (auto & plan : plans)
{
SortDescription sort_description;
/// Drop temporary columns, added by 'sorting_key_prefix_expr'
out_projection = createProjection(plan->getCurrentDataStream().header, data);
if (pipe.numOutputPorts() > 1 || force_sorting_key_transform)
{
for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j)
sort_description.emplace_back(metadata_snapshot->getSortingKey().column_names[j],
input_order_info->direction, 1);
auto expression_step = std::make_unique<ExpressionStep>(
plan->getCurrentDataStream(),
sorting_key_prefix_expr);
/// Drop temporary columns, added by 'sorting_key_prefix_expr'
out_projection = createProjection(pipe, data);
pipe.addSimpleTransform([sorting_key_prefix_expr](const Block & header)
{
return std::make_shared<ExpressionTransform>(header, sorting_key_prefix_expr);
});
}
expression_step->setStepDescription("Calculate sorting key prefix");
plan->addStep(std::move(expression_step));
if (pipe.numOutputPorts() > 1)
{
pipe.addTransform(std::make_shared<MergingSortedTransform>(
pipe.getHeader(), pipe.numOutputPorts(), sort_description, max_block_size));
}
auto merging_sorted = std::make_unique<MergingSortedStep>(
plan->getCurrentDataStream(),
sort_description,
max_block_size);
merging_sorted->setStepDescription("Merge sorting mark ranges");
plan->addStep(std::move(merging_sorted));
}
}
return Pipe::unitePipes(std::move(res));
if (plans.size() == 1)
return std::move(plans.front());
DataStreams input_streams;
for (const auto & plan : plans)
input_streams.emplace_back(plan->getCurrentDataStream());
const auto & common_header = plans.front()->getCurrentDataStream().header;
auto union_step = std::make_unique<UnionStep>(std::move(input_streams), common_header);
auto plan = std::make_unique<QueryPlan>();
plan->unitePlans(std::move(union_step), std::move(plans));
return plan;
}
Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal(
QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal(
RangesInDataParts && parts,
size_t num_streams,
const Names & column_names,
@ -1316,11 +1327,11 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal(
parts_to_merge_ranges.push_back(parts.end());
}
Pipes partition_pipes;
std::vector<QueryPlanPtr> partition_plans;
for (size_t range_index = 0; range_index < parts_to_merge_ranges.size() - 1; ++range_index)
{
Pipe pipe;
QueryPlanPtr plan;
{
Pipes pipes;
@ -1346,12 +1357,17 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal(
pipes.emplace_back(std::move(source_processor));
}
pipe = Pipe::unitePipes(std::move(pipes));
}
if (pipes.empty())
continue;
/// Drop temporary columns, added by 'sorting_key_expr'
if (!out_projection)
out_projection = createProjection(pipe, data);
auto pipe = Pipe::unitePipes(std::move(pipes));
/// Drop temporary columns, added by 'sorting_key_expr'
if (!out_projection)
out_projection = createProjection(pipe.getHeader(), data);
plan = createPlanFromPipe(std::move(pipe), "with final");
}
/// If do_not_merge_across_partitions_select_final is true and there is only one part in partition
/// with level > 0 then we won't postprocess this part
@ -1359,14 +1375,16 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal(
std::distance(parts_to_merge_ranges[range_index], parts_to_merge_ranges[range_index + 1]) == 1 &&
parts_to_merge_ranges[range_index]->data_part->info.level > 0)
{
partition_pipes.emplace_back(std::move(pipe));
partition_plans.emplace_back(std::move(plan));
continue;
}
pipe.addSimpleTransform([&metadata_snapshot](const Block & header)
{
return std::make_shared<ExpressionTransform>(header, metadata_snapshot->getSortingKey().expression);
});
auto expression_step = std::make_unique<ExpressionStep>(
plan->getCurrentDataStream(),
metadata_snapshot->getSortingKey().expression);
expression_step->setStepDescription("Calculate sorting key expression");
plan->addStep(std::move(expression_step));
Names sort_columns = metadata_snapshot->getSortingKeyColumns();
SortDescription sort_description;
@ -1375,111 +1393,40 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal(
Names partition_key_columns = metadata_snapshot->getPartitionKey().column_names;
Block header = pipe.getHeader();
const auto & header = plan->getCurrentDataStream().header;
for (size_t i = 0; i < sort_columns_size; ++i)
sort_description.emplace_back(header.getPositionByName(sort_columns[i]), 1, 1);
auto get_merging_processor = [&]() -> MergingTransformPtr
{
switch (data.merging_params.mode)
{
case MergeTreeData::MergingParams::Ordinary:
{
return std::make_shared<MergingSortedTransform>(header, pipe.numOutputPorts(), sort_description, max_block_size);
}
auto final_step = std::make_unique<MergingFinal>(
plan->getCurrentDataStream(),
std::min<size_t>(num_streams, settings.max_final_threads),
sort_description,
data.merging_params,
partition_key_columns,
max_block_size);
case MergeTreeData::MergingParams::Collapsing:
return std::make_shared<CollapsingSortedTransform>(
header, pipe.numOutputPorts(), sort_description, data.merging_params.sign_column, true, max_block_size);
final_step->setStepDescription("Merge rows for FINAL");
plan->addStep(std::move(final_step));
case MergeTreeData::MergingParams::Summing:
return std::make_shared<SummingSortedTransform>(
header,
pipe.numOutputPorts(),
sort_description,
data.merging_params.columns_to_sum,
partition_key_columns,
max_block_size);
case MergeTreeData::MergingParams::Aggregating:
return std::make_shared<AggregatingSortedTransform>(header, pipe.numOutputPorts(), sort_description, max_block_size);
case MergeTreeData::MergingParams::Replacing:
return std::make_shared<ReplacingSortedTransform>(
header, pipe.numOutputPorts(), sort_description, data.merging_params.version_column, max_block_size);
case MergeTreeData::MergingParams::VersionedCollapsing:
return std::make_shared<VersionedCollapsingTransform>(
header, pipe.numOutputPorts(), sort_description, data.merging_params.sign_column, max_block_size);
case MergeTreeData::MergingParams::Graphite:
throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR);
}
__builtin_unreachable();
};
if (num_streams <= 1 || sort_description.empty())
{
pipe.addTransform(get_merging_processor());
partition_pipes.emplace_back(std::move(pipe));
continue;
}
ColumnNumbers key_columns;
key_columns.reserve(sort_description.size());
for (auto & desc : sort_description)
{
if (!desc.column_name.empty())
key_columns.push_back(header.getPositionByName(desc.column_name));
else
key_columns.emplace_back(desc.column_number);
}
pipe.addSimpleTransform([&](const Block & stream_header)
{
return std::make_shared<AddingSelectorTransform>(stream_header, num_streams, key_columns);
});
pipe.transform([&](OutputPortRawPtrs ports)
{
Processors processors;
std::vector<OutputPorts::iterator> output_ports;
processors.reserve(ports.size() + num_streams);
output_ports.reserve(ports.size());
for (auto & port : ports)
{
auto copier = std::make_shared<CopyTransform>(header, num_streams);
connect(*port, copier->getInputPort());
output_ports.emplace_back(copier->getOutputs().begin());
processors.emplace_back(std::move(copier));
}
for (size_t i = 0; i < num_streams; ++i)
{
auto merge = get_merging_processor();
merge->setSelectorPosition(i);
auto input = merge->getInputs().begin();
/// Connect i-th merge with i-th input port of every copier.
for (size_t j = 0; j < ports.size(); ++j)
{
connect(*output_ports[j], *input);
++output_ports[j];
++input;
}
processors.emplace_back(std::move(merge));
}
return processors;
});
partition_pipes.emplace_back(std::move(pipe));
partition_plans.emplace_back(std::move(plan));
}
return Pipe::unitePipes(std::move(partition_pipes));
if (partition_plans.empty())
return {};
if (partition_plans.size() == 1)
return std::move(partition_plans.front());
auto result_header = partition_plans.front()->getCurrentDataStream().header;
DataStreams input_streams;
for (const auto & partition_plan : partition_plans)
input_streams.push_back(partition_plan->getCurrentDataStream());
auto union_step = std::make_unique<UnionStep>(std::move(input_streams), result_header);
union_step->setStepDescription("Unite sources after FINAL");
QueryPlanPtr plan = std::make_unique<QueryPlan>();
plan->unitePlans(std::move(union_step), std::move(partition_plans));
return plan;
}
/// Calculates a set of mark ranges, that could possibly contain keys, required by condition.

10
src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
View File

@ -24,7 +24,7 @@ public:
*/
using PartitionIdToMaxBlock = std::unordered_map<String, Int64>;
Pipe read(
QueryPlanPtr read(
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
const SelectQueryInfo & query_info,
@ -33,7 +33,7 @@ public:
unsigned num_streams,
const PartitionIdToMaxBlock * max_block_numbers_to_read = nullptr) const;
Pipe readFromParts(
QueryPlanPtr readFromParts(
MergeTreeData::DataPartsVector parts,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
@ -48,7 +48,7 @@ private:
Poco::Logger * log;
Pipe spreadMarkRangesAmongStreams(
QueryPlanPtr spreadMarkRangesAmongStreams(
RangesInDataParts && parts,
size_t num_streams,
const Names & column_names,
@ -61,7 +61,7 @@ private:
const MergeTreeReaderSettings & reader_settings) const;
/// out_projection - save projection only with columns, requested to read
Pipe spreadMarkRangesAmongStreamsWithOrder(
QueryPlanPtr spreadMarkRangesAmongStreamsWithOrder(
RangesInDataParts && parts,
size_t num_streams,
const Names & column_names,
@ -75,7 +75,7 @@ private:
const MergeTreeReaderSettings & reader_settings,
ExpressionActionsPtr & out_projection) const;
Pipe spreadMarkRangesAmongStreamsFinal(
QueryPlanPtr spreadMarkRangesAmongStreamsFinal(
RangesInDataParts && parts,
size_t num_streams,
const Names & column_names,

5
src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h
View File

@ -35,9 +35,10 @@ struct ReplicatedMergeTreeMutationEntry
/// Replica which initiated mutation
String source_replica;
/// Accured numbers of blocks
/// Acquired block numbers
/// partition_id -> block_number
std::map<String, Int64> block_numbers;
using BlockNumbersType = std::map<String, Int64>;
BlockNumbersType block_numbers;
/// Mutation commands which will give to MUTATE_PART entries
MutationCommands commands;

19
src/Storages/MergeTree/StorageFromMergeTreeDataPart.h
View File

@ -3,6 +3,8 @@
#include <Storages/IStorage.h>
#include <Storages/MergeTree/IMergeTreeDataPart.h>
#include <Storages/MergeTree/MergeTreeDataSelectExecutor.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Processors/QueryPipeline.h>
#include <Core/Defines.h>
#include <ext/shared_ptr_helper.h>
@ -27,8 +29,11 @@ public:
size_t max_block_size,
unsigned num_streams) override
{
return MergeTreeDataSelectExecutor(part->storage)
.readFromParts({part}, column_names, metadata_snapshot, query_info, context, max_block_size, num_streams);
QueryPlan query_plan =
std::move(*MergeTreeDataSelectExecutor(part->storage)
.readFromParts({part}, column_names, metadata_snapshot, query_info, context, max_block_size, num_streams));
return query_plan.convertToPipe();
}
@ -45,6 +50,16 @@ public:
return part->storage.getVirtuals();
}
String getPartitionId() const
{
return part->info.partition_id;
}
String getPartitionIDFromQuery(const ASTPtr & ast, const Context & context) const
{
return part->storage.getPartitionIDFromQuery(ast, context);
}
protected:
StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_)
: IStorage(getIDFromPart(part_))

7
src/Storages/MutationCommands.cpp
View File

@ -2,11 +2,13 @@
#include <IO/Operators.h>
#include <Parsers/formatAST.h>
#include <Parsers/ExpressionListParsers.h>
#include <Parsers/ASTColumnDeclaration.h>
#include <Parsers/ParserAlterQuery.h>
#include <Parsers/parseQuery.h>
#include <Parsers/ASTAssignment.h>
#include <Parsers/ASTColumnDeclaration.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Common/typeid_cast.h>
#include <Common/quoteString.h>
#include <Core/Defines.h>
@ -32,6 +34,7 @@ std::optional<MutationCommand> MutationCommand::parse(ASTAlterCommand * command,
res.ast = command->ptr();
res.type = DELETE;
res.predicate = command->predicate;
res.partition = command->partition;
return res;
}
else if (command->type == ASTAlterCommand::UPDATE)
@ -40,6 +43,7 @@ std::optional<MutationCommand> MutationCommand::parse(ASTAlterCommand * command,
res.ast = command->ptr();
res.type = UPDATE;
res.predicate = command->predicate;
res.partition = command->partition;
for (const ASTPtr & assignment_ast : command->update_assignments->children)
{
const auto & assignment = assignment_ast->as<ASTAssignment &>();
@ -124,6 +128,7 @@ std::shared_ptr<ASTAlterCommandList> MutationCommands::ast() const
return res;
}
void MutationCommands::writeText(WriteBuffer & out) const
{
std::stringstream commands_ss;

4
src/Storages/MutationCommands.h
View File

@ -43,8 +43,10 @@ struct MutationCommand
/// Columns with corresponding actions
std::unordered_map<String, ASTPtr> column_to_update_expression;
/// For MATERIALIZE INDEX
/// For MATERIALIZE INDEX.
String index_name;
/// For MATERIALIZE INDEX, UPDATE and DELETE.
ASTPtr partition;
/// For reads, drops and etc.

164
src/Storages/StorageBuffer.cpp
View File

@ -4,7 +4,7 @@
#include <Interpreters/InterpreterAlterQuery.h>
#include <Interpreters/castColumn.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Processors/Transforms/AddingMissedTransform.h>
#include <Processors/QueryPlan/AddingMissedStep.h>
#include <DataStreams/IBlockInputStream.h>
#include <Storages/StorageBuffer.h>
#include <Storages/StorageFactory.h>
@ -22,10 +22,13 @@
#include <common/logger_useful.h>
#include <common/getThreadId.h>
#include <ext/range.h>
#include <Processors/Transforms/ConvertingTransform.h>
#include <Processors/QueryPlan/ConvertingStep.h>
#include <Processors/Transforms/FilterTransform.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <Processors/Sources/SourceFromInputStream.h>
#include <Processors/QueryPlan/SettingQuotaAndLimitsStep.h>
#include <Processors/QueryPlan/ReadFromPreparedSource.h>
#include <Processors/QueryPlan/UnionStep.h>
namespace ProfileEvents
@ -147,6 +150,21 @@ QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage(const Context
Pipe StorageBuffer::read(
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
const size_t max_block_size,
const unsigned num_streams)
{
QueryPlan plan;
read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
return plan.convertToPipe();
}
void StorageBuffer::read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
@ -155,8 +173,6 @@ Pipe StorageBuffer::read(
size_t max_block_size,
unsigned num_streams)
{
Pipe pipe_from_dst;
if (destination_id)
{
auto destination = DatabaseCatalog::instance().getTable(destination_id, context);
@ -182,8 +198,8 @@ Pipe StorageBuffer::read(
query_info.input_order_info = query_info.order_optimizer->getInputOrder(destination_metadata_snapshot);
/// The destination table has the same structure of the requested columns and we can simply read blocks from there.
pipe_from_dst = destination->read(
column_names, destination_metadata_snapshot, query_info,
destination->read(
query_plan, column_names, destination_metadata_snapshot, query_info,
context, processed_stage, max_block_size, num_streams);
}
else
@ -217,29 +233,49 @@ Pipe StorageBuffer::read(
}
else
{
pipe_from_dst = destination->read(
columns_intersection, destination_metadata_snapshot, query_info,
context, processed_stage, max_block_size, num_streams);
destination->read(
query_plan, columns_intersection, destination_metadata_snapshot, query_info,
context, processed_stage, max_block_size, num_streams);
if (!pipe_from_dst.empty())
if (query_plan.isInitialized())
{
pipe_from_dst.addSimpleTransform([&](const Block & stream_header)
{
return std::make_shared<AddingMissedTransform>(stream_header, header_after_adding_defaults,
metadata_snapshot->getColumns(), context);
});
pipe_from_dst.addSimpleTransform([&](const Block & stream_header)
{
return std::make_shared<ConvertingTransform>(
stream_header, header, ConvertingTransform::MatchColumnsMode::Name);
});
auto adding_missed = std::make_unique<AddingMissedStep>(
query_plan.getCurrentDataStream(),
header_after_adding_defaults,
metadata_snapshot->getColumns(), context);
adding_missed->setStepDescription("Add columns missing in destination table");
query_plan.addStep(std::move(adding_missed));
auto converting = std::make_unique<ConvertingStep>(
query_plan.getCurrentDataStream(),
header);
converting->setStepDescription("Convert destination table columns to Buffer table structure");
query_plan.addStep(std::move(converting));
}
}
}
pipe_from_dst.addTableLock(destination_lock);
pipe_from_dst.addStorageHolder(destination);
if (query_plan.isInitialized())
{
StreamLocalLimits limits;
SizeLimits leaf_limits;
/// Add table lock for destination table.
auto adding_limits_and_quota = std::make_unique<SettingQuotaAndLimitsStep>(
query_plan.getCurrentDataStream(),
destination,
std::move(destination_lock),
limits,
leaf_limits,
nullptr,
nullptr);
adding_limits_and_quota->setStepDescription("Lock destination table for Buffer");
query_plan.addStep(std::move(adding_limits_and_quota));
}
}
Pipe pipe_from_buffers;
@ -252,49 +288,73 @@ Pipe StorageBuffer::read(
pipe_from_buffers = Pipe::unitePipes(std::move(pipes_from_buffers));
}
/// Convert pipes from table to structure from buffer.
if (!pipe_from_buffers.empty() && !pipe_from_dst.empty()
&& !blocksHaveEqualStructure(pipe_from_buffers.getHeader(), pipe_from_dst.getHeader()))
{
pipe_from_dst.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ConvertingTransform>(
header,
pipe_from_buffers.getHeader(),
ConvertingTransform::MatchColumnsMode::Name);
});
}
if (pipe_from_buffers.empty())
return;
QueryPlan buffers_plan;
/** If the sources from the table were processed before some non-initial stage of query execution,
* then sources from the buffers must also be wrapped in the processing pipeline before the same stage.
*/
if (processed_stage > QueryProcessingStage::FetchColumns)
pipe_from_buffers = QueryPipeline::getPipe(
InterpreterSelectQuery(query_info.query, context, std::move(pipe_from_buffers),
SelectQueryOptions(processed_stage)).execute().pipeline);
if (query_info.prewhere_info)
{
pipe_from_buffers.addSimpleTransform([&](const Block & header)
{
return std::make_shared<FilterTransform>(
header, query_info.prewhere_info->prewhere_actions,
query_info.prewhere_info->prewhere_column_name, query_info.prewhere_info->remove_prewhere_column);
});
if (query_info.prewhere_info->alias_actions)
auto interpreter = InterpreterSelectQuery(
query_info.query, context, std::move(pipe_from_buffers),
SelectQueryOptions(processed_stage));
interpreter.buildQueryPlan(buffers_plan);
}
else
{
if (query_info.prewhere_info)
{
pipe_from_buffers.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ExpressionTransform>(header, query_info.prewhere_info->alias_actions);
return std::make_shared<FilterTransform>(
header, query_info.prewhere_info->prewhere_actions,
query_info.prewhere_info->prewhere_column_name, query_info.prewhere_info->remove_prewhere_column);
});
if (query_info.prewhere_info->alias_actions)
{
pipe_from_buffers.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ExpressionTransform>(header, query_info.prewhere_info->alias_actions);
});
}
}
auto read_from_buffers = std::make_unique<ReadFromPreparedSource>(std::move(pipe_from_buffers));
read_from_buffers->setStepDescription("Read from buffers of Buffer table");
buffers_plan.addStep(std::move(read_from_buffers));
}
Pipes pipes;
pipes.emplace_back(std::move(pipe_from_dst));
pipes.emplace_back(std::move(pipe_from_buffers));
return Pipe::unitePipes(std::move(pipes));
if (!query_plan.isInitialized())
{
query_plan = std::move(buffers_plan);
return;
}
auto result_header = buffers_plan.getCurrentDataStream().header;
/// Convert structure from table to structure from buffer.
if (!blocksHaveEqualStructure(query_plan.getCurrentDataStream().header, result_header))
{
auto converting = std::make_unique<ConvertingStep>(query_plan.getCurrentDataStream(), result_header);
query_plan.addStep(std::move(converting));
}
DataStreams input_streams;
input_streams.emplace_back(query_plan.getCurrentDataStream());
input_streams.emplace_back(buffers_plan.getCurrentDataStream());
std::vector<std::unique_ptr<QueryPlan>> plans;
plans.emplace_back(std::make_unique<QueryPlan>(std::move(query_plan)));
plans.emplace_back(std::make_unique<QueryPlan>(std::move(buffers_plan)));
query_plan = QueryPlan();
auto union_step = std::make_unique<UnionStep>(std::move(input_streams), result_header);
union_step->setStepDescription("Unite sources from Buffer table");
query_plan.unitePlans(std::move(union_step), std::move(plans));
}

10
src/Storages/StorageBuffer.h
View File

@ -65,6 +65,16 @@ public:
size_t max_block_size,
unsigned num_streams) override;
void read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams) override;
bool supportsParallelInsert() const override { return true; }
BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override;

18
src/Storages/StorageDistributed.cpp
View File

@ -482,6 +482,21 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(
}
Pipe StorageDistributed::read(
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
const size_t max_block_size,
const unsigned num_streams)
{
QueryPlan plan;
read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
return plan.convertToPipe();
}
void StorageDistributed::read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
@ -508,7 +523,8 @@ Pipe StorageDistributed::read(
: ClusterProxy::SelectStreamFactory(
header, processed_stage, StorageID{remote_database, remote_table}, scalars, has_virtual_shard_num_column, context.getExternalTables());
return ClusterProxy::executeQuery(select_stream_factory, log, modified_query_ast, context, query_info);
ClusterProxy::executeQuery(query_plan, select_stream_factory, log,
modified_query_ast, context, query_info);
}

10
src/Storages/StorageDistributed.h
View File

@ -62,6 +62,16 @@ public:
size_t max_block_size,
unsigned num_streams) override;
void read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t /*max_block_size*/,
unsigned /*num_streams*/) override;
bool supportsParallelInsert() const override { return true; }
BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override;

39
src/Storages/StorageMaterializedView.cpp
View File

@ -22,6 +22,7 @@
#include <Common/typeid_cast.h>
#include <Processors/Sources/SourceFromInputStream.h>
#include <Processors/QueryPlan/SettingQuotaAndLimitsStep.h>
namespace DB
@ -107,6 +108,21 @@ QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage(cons
}
Pipe StorageMaterializedView::read(
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
const size_t max_block_size,
const unsigned num_streams)
{
QueryPlan plan;
read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
return plan.convertToPipe();
}
void StorageMaterializedView::read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & /*metadata_snapshot*/,
SelectQueryInfo & query_info,
@ -122,11 +138,26 @@ Pipe StorageMaterializedView::read(
if (query_info.order_optimizer)
query_info.input_order_info = query_info.order_optimizer->getInputOrder(metadata_snapshot);
Pipe pipe = storage->read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
pipe.addTableLock(lock);
pipe.addStorageHolder(storage);
storage->read(query_plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
return pipe;
if (query_plan.isInitialized())
{
StreamLocalLimits limits;
SizeLimits leaf_limits;
/// Add table lock for destination table.
auto adding_limits_and_quota = std::make_unique<SettingQuotaAndLimitsStep>(
query_plan.getCurrentDataStream(),
storage,
std::move(lock),
limits,
leaf_limits,
nullptr,
nullptr);
adding_limits_and_quota->setStepDescription("Lock destination table for Buffer");
query_plan.addStep(std::move(adding_limits_and_quota));
}
}
BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context)

10
src/Storages/StorageMaterializedView.h
View File

@ -81,6 +81,16 @@ public:
size_t max_block_size,
unsigned num_streams) override;
void read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams) override;
Strings getDataPaths() const override;
private:

9
src/Storages/StorageMerge.cpp
View File

@ -332,10 +332,13 @@ Pipe StorageMerge::createSources(
if (has_table_virtual_column)
{
pipe.addSimpleTransform([name = table_name](const Block & stream_header)
ColumnWithTypeAndName column;
column.name = "_table";
column.type = std::make_shared<DataTypeString>();
column.column = column.type->createColumnConst(0, Field(table_name));
pipe.addSimpleTransform([&](const Block & stream_header)
{
return std::make_shared<AddingConstColumnTransform<String>>(
stream_header, std::make_shared<DataTypeString>(), name, "_table");
return std::make_shared<AddingConstColumnTransform>(stream_header, column);
});
}

21
src/Storages/StorageMergeTree.cpp
View File

@ -173,17 +173,32 @@ StorageMergeTree::~StorageMergeTree()
shutdown();
}
Pipe StorageMergeTree::read(
void StorageMergeTree::read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum /*processed_stage*/,
size_t max_block_size,
unsigned num_streams)
{
if (auto plan = reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams))
query_plan = std::move(*plan);
}
Pipe StorageMergeTree::read(
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
const size_t max_block_size,
const unsigned num_streams)
{
return reader.read(column_names, metadata_snapshot, query_info,
context, max_block_size, num_streams);
QueryPlan plan;
read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
return plan.convertToPipe();
}
std::optional<UInt64> StorageMergeTree::totalRows() const

10
src/Storages/StorageMergeTree.h
View File

@ -46,6 +46,16 @@ public:
size_t max_block_size,
unsigned num_streams) override;
void read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & /*metadata_snapshot*/,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams) override;
std::optional<UInt64> totalRows() const override;
std::optional<UInt64> totalRowsByPartitionPredicate(const SelectQueryInfo &, const Context &) const override;
std::optional<UInt64> totalBytes() const override;

148
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -3643,7 +3643,9 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg
return max_added_blocks;
}
Pipe StorageReplicatedMergeTree::read(
void StorageReplicatedMergeTree::read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
@ -3660,10 +3662,27 @@ Pipe StorageReplicatedMergeTree::read(
if (context.getSettingsRef().select_sequential_consistency)
{
auto max_added_blocks = getMaxAddedBlocks();
return reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams, &max_added_blocks);
if (auto plan = reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams, &max_added_blocks))
query_plan = std::move(*plan);
return;
}
return reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams);
if (auto plan = reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams))
query_plan = std::move(*plan);
}
Pipe StorageReplicatedMergeTree::read(
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
const size_t max_block_size,
const unsigned num_streams)
{
QueryPlan plan;
read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
return plan.convertToPipe();
}
@ -3915,6 +3934,60 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer
}
std::set<String> StorageReplicatedMergeTree::getPartitionIdsAffectedByCommands(
const MutationCommands & commands, const Context & query_context) const
{
std::set<String> affected_partition_ids;
for (const auto & command : commands)
{
if (!command.partition)
{
affected_partition_ids.clear();
break;
}
affected_partition_ids.insert(
getPartitionIDFromQuery(command.partition, query_context)
);
}
return affected_partition_ids;
}
PartitionBlockNumbersHolder StorageReplicatedMergeTree::allocateBlockNumbersInAffectedPartitions(
const MutationCommands & commands, const Context & query_context, const zkutil::ZooKeeperPtr & zookeeper) const
{
const std::set<String> mutation_affected_partition_ids = getPartitionIdsAffectedByCommands(commands, query_context);
if (mutation_affected_partition_ids.size() == 1)
{
const auto & affected_partition_id = *mutation_affected_partition_ids.cbegin();
auto block_number_holder = allocateBlockNumber(affected_partition_id, zookeeper);
if (!block_number_holder.has_value())
return {};
auto block_number = block_number_holder->getNumber(); /// Avoid possible UB due to std::move
return {{{affected_partition_id, block_number}}, std::move(block_number_holder)};
}
else
{
/// TODO: Implement optimal block number aqcuisition algorithm in multiple (but not all) partitions
EphemeralLocksInAllPartitions lock_holder(
zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper);
PartitionBlockNumbersHolder::BlockNumbersType block_numbers;
for (const auto & lock : lock_holder.getLocks())
{
if (mutation_affected_partition_ids.empty() || mutation_affected_partition_ids.count(lock.partition_id))
block_numbers[lock.partition_id] = lock.number;
}
return {std::move(block_numbers), std::move(lock_holder)};
}
}
void StorageReplicatedMergeTree::alter(
const AlterCommands & commands, const Context & query_context, TableLockHolder & table_lock_holder)
{
@ -3942,7 +4015,7 @@ void StorageReplicatedMergeTree::alter(
return queryToString(query);
};
auto zookeeper = getZooKeeper();
const auto zookeeper = getZooKeeper();
std::optional<ReplicatedMergeTreeLogEntryData> alter_entry;
std::optional<String> mutation_znode;
@ -3953,10 +4026,6 @@ void StorageReplicatedMergeTree::alter(
alter_entry.emplace();
mutation_znode.reset();
/// We can safely read structure, because we guarded with alter_intention_lock
if (is_readonly)
throw Exception("Can't ALTER readonly table", ErrorCodes::TABLE_IS_READ_ONLY);
auto current_metadata = getInMemoryMetadataPtr();
StorageInMemoryMetadata future_metadata = *current_metadata;
@ -4029,27 +4098,23 @@ void StorageReplicatedMergeTree::alter(
ops.emplace_back(zkutil::makeCreateRequest(
zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential));
std::optional<EphemeralLocksInAllPartitions> lock_holder;
/// Now we will prepare mutations record.
/// This code pretty same with mutate() function but process results slightly differently.
PartitionBlockNumbersHolder partition_block_numbers_holder;
if (alter_entry->have_mutation)
{
String mutations_path = zookeeper_path + "/mutations";
const String mutations_path(zookeeper_path + "/mutations");
ReplicatedMergeTreeMutationEntry mutation_entry;
mutation_entry.source_replica = replica_name;
mutation_entry.commands = maybe_mutation_commands;
mutation_entry.alter_version = new_metadata_version;
mutation_entry.source_replica = replica_name;
mutation_entry.commands = std::move(maybe_mutation_commands);
Coordination::Stat mutations_stat;
zookeeper->get(mutations_path, &mutations_stat);
lock_holder.emplace(
zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper);
for (const auto & lock : lock_holder->getLocks())
mutation_entry.block_numbers[lock.partition_id] = lock.number;
partition_block_numbers_holder =
allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper);
mutation_entry.block_numbers = partition_block_numbers_holder.getBlockNumbers();
mutation_entry.create_time = time(nullptr);
ops.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version));
@ -4060,6 +4125,11 @@ void StorageReplicatedMergeTree::alter(
Coordination::Responses results;
Coordination::Error rc = zookeeper->tryMulti(ops, results);
/// For the sake of constitency with mechanics of concurrent background process of assigning parts merge tasks
/// this placeholder must be held up until the moment of committing into ZK of the mutation entry
/// See ReplicatedMergeTreeMergePredicate::canMergeTwoParts() method
partition_block_numbers_holder.reset();
if (rc == Coordination::Error::ZOK)
{
if (alter_entry->have_mutation)
@ -4398,7 +4468,7 @@ void StorageReplicatedMergeTree::rename(const String & new_path_to_table_data, c
}
bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path)
bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) const
{
{
std::lock_guard lock(existing_nodes_cache_mutex);
@ -4420,7 +4490,7 @@ bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path)
std::optional<EphemeralLockInZooKeeper>
StorageReplicatedMergeTree::allocateBlockNumber(
const String & partition_id, zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_block_id_path)
const String & partition_id, const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_block_id_path) const
{
/// Lets check for duplicates in advance, to avoid superfluous block numbers allocation
Coordination::Requests deduplication_check_ops;
@ -5063,44 +5133,46 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const
/// After all needed parts are mutated (i.e. all active parts have the mutation version greater than
/// the version of this mutation), the mutation is considered done and can be deleted.
ReplicatedMergeTreeMutationEntry entry;
entry.source_replica = replica_name;
entry.commands = commands;
ReplicatedMergeTreeMutationEntry mutation_entry;
mutation_entry.source_replica = replica_name;
mutation_entry.commands = commands;
String mutations_path = zookeeper_path + "/mutations";
const String mutations_path = zookeeper_path + "/mutations";
const auto zookeeper = getZooKeeper();
/// Update the mutations_path node when creating the mutation and check its version to ensure that
/// nodes for mutations are created in the same order as the corresponding block numbers.
/// Should work well if the number of concurrent mutation requests is small.
while (true)
{
auto zookeeper = getZooKeeper();
Coordination::Stat mutations_stat;
zookeeper->get(mutations_path, &mutations_stat);
EphemeralLocksInAllPartitions block_number_locks(
zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper);
PartitionBlockNumbersHolder partition_block_numbers_holder =
allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper);
for (const auto & lock : block_number_locks.getLocks())
entry.block_numbers[lock.partition_id] = lock.number;
entry.create_time = time(nullptr);
mutation_entry.block_numbers = partition_block_numbers_holder.getBlockNumbers();
mutation_entry.create_time = time(nullptr);
/// The following version check guarantees the linearizability property for any pair of mutations:
/// mutation with higher sequence number is guaranteed to have higher block numbers in every partition
/// (and thus will be applied strictly according to sequence numbers of mutations)
Coordination::Requests requests;
requests.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version));
requests.emplace_back(zkutil::makeCreateRequest(
mutations_path + "/", entry.toString(), zkutil::CreateMode::PersistentSequential));
mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential));
Coordination::Responses responses;
Coordination::Error rc = zookeeper->tryMulti(requests, responses);
partition_block_numbers_holder.reset();
if (rc == Coordination::Error::ZOK)
{
const String & path_created =
dynamic_cast<const Coordination::CreateResponse *>(responses[1].get())->path_created;
entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1);
LOG_TRACE(log, "Created mutation with ID {}", entry.znode_name);
mutation_entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1);
LOG_TRACE(log, "Created mutation with ID {}", mutation_entry.znode_name);
break;
}
else if (rc == Coordination::Error::ZBADVERSION)
@ -5112,7 +5184,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const
throw Coordination::Exception("Unable to create a mutation znode", rc);
}
waitMutation(entry.znode_name, query_context.getSettingsRef().mutations_sync);
waitMutation(mutation_entry.znode_name, query_context.getSettingsRef().mutations_sync);
}
void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t mutations_sync) const

25
src/Storages/StorageReplicatedMergeTree.h
View File

@ -96,6 +96,16 @@ public:
size_t max_block_size,
unsigned num_streams) override;
void read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & /*metadata_snapshot*/,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams) override;
std::optional<UInt64> totalRows() const override;
std::optional<UInt64> totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const override;
std::optional<UInt64> totalBytes() const override;
@ -506,8 +516,8 @@ private:
/// Creates new block number if block with such block_id does not exist
std::optional<EphemeralLockInZooKeeper> allocateBlockNumber(
const String & partition_id, zkutil::ZooKeeperPtr & zookeeper,
const String & zookeeper_block_id_path = "");
const String & partition_id, const zkutil::ZooKeeperPtr & zookeeper,
const String & zookeeper_block_id_path = "") const;
/** Wait until all replicas, including this, execute the specified action from the log.
* If replicas are added at the same time, it can not wait the added replica .
@ -531,9 +541,9 @@ private:
bool getFakePartCoveringAllPartsInPartition(const String & partition_id, MergeTreePartInfo & part_info, bool for_replace_partition = false);
/// Check for a node in ZK. If it is, remember this information, and then immediately answer true.
std::unordered_set<std::string> existing_nodes_cache;
std::mutex existing_nodes_cache_mutex;
bool existsNodeCached(const std::string & path);
mutable std::unordered_set<std::string> existing_nodes_cache;
mutable std::mutex existing_nodes_cache_mutex;
bool existsNodeCached(const std::string & path) const;
void getClearBlocksInPartitionOps(Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num);
/// Remove block IDs from `blocks/` in ZooKeeper for the given partition ID in the given block number range.
@ -565,6 +575,11 @@ private:
MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const override;
void startBackgroundMovesIfNeeded() override;
std::set<String> getPartitionIdsAffectedByCommands(const MutationCommands & commands, const Context & query_context) const;
PartitionBlockNumbersHolder allocateBlockNumbersInAffectedPartitions(
const MutationCommands & commands, const Context & query_context, const zkutil::ZooKeeperPtr & zookeeper) const;
protected:
/** If not 'attach', either creates a new table in ZK, or adds a replica to an existing table.
*/

46
src/Storages/StorageView.cpp
View File

@ -16,6 +16,9 @@
#include <Processors/Pipe.h>
#include <Processors/Transforms/MaterializingTransform.h>
#include <Processors/Transforms/ConvertingTransform.h>
#include <Processors/QueryPlan/MaterializingStep.h>
#include <Processors/QueryPlan/ConvertingStep.h>
#include <Processors/QueryPlan/SettingQuotaAndLimitsStep.h>
namespace DB
{
@ -52,12 +55,25 @@ Pipe StorageView::read(
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum /*processed_stage*/,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
QueryProcessingStage::Enum processed_stage,
const size_t max_block_size,
const unsigned num_streams)
{
Pipes pipes;
QueryPlan plan;
read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
return plan.convertToPipe();
}
void StorageView::read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum /*processed_stage*/,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
{
ASTPtr current_inner_query = metadata_snapshot->getSelectQuery().inner_query;
if (query_info.view_query)
@ -68,25 +84,19 @@ Pipe StorageView::read(
}
InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names);
auto pipeline = interpreter.execute().pipeline;
interpreter.buildQueryPlan(query_plan);
/// It's expected that the columns read from storage are not constant.
/// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery.
pipeline.addSimpleTransform([](const Block & header)
{
return std::make_shared<MaterializingTransform>(header);
});
auto materializing = std::make_unique<MaterializingStep>(query_plan.getCurrentDataStream());
materializing->setStepDescription("Materialize constants after VIEW subquery");
query_plan.addStep(std::move(materializing));
/// And also convert to expected structure.
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ConvertingTransform>(
header, metadata_snapshot->getSampleBlockForColumns(
column_names, getVirtuals(), getStorageID()), ConvertingTransform::MatchColumnsMode::Name);
});
return QueryPipeline::getPipe(std::move(pipeline));
auto header = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID());
auto converting = std::make_unique<ConvertingStep>(query_plan.getCurrentDataStream(), header);
converting->setStepDescription("Convert VIEW subquery result to VIEW table structure");
query_plan.addStep(std::move(converting));
}
static ASTTableExpression * getFirstTableExpression(ASTSelectQuery & select_query)

10
src/Storages/StorageView.h
View File

@ -30,6 +30,16 @@ public:
size_t max_block_size,
unsigned num_streams) override;
void read(
QueryPlan & query_plan,
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams) override;
void replaceWithSubquery(ASTSelectQuery & select_query, ASTPtr & view_name, const StorageMetadataPtr & metadata_snapshot) const
{
replaceWithSubquery(select_query, metadata_snapshot->getSelectQuery().inner_query->clone(), view_name);

0
tests/integration/test_mutations_in_partitions_of_merge_tree/__init__.py Normal file
View File

16
tests/integration/test_mutations_in_partitions_of_merge_tree/configs/cluster.xml Normal file
View File

@ -0,0 +1,16 @@
<yandex>
<remote_servers>
<test_cluster>
<shard>
<replica>
<host>node1</host>
<port>9000</port>
</replica>
<replica>
<host>node2</host>
<port>9000</port>
</replica>
</shard>
</test_cluster>
</remote_servers>
</yandex>

17
tests/integration/test_mutations_in_partitions_of_merge_tree/configs/logs_config.xml Normal file
View File

@ -0,0 +1,17 @@
<yandex>
<shutdown_wait_unfinished>3</shutdown_wait_unfinished>
<logger>
<level>trace</level>
<log>/var/log/clickhouse-server/log.log</log>
<errorlog>/var/log/clickhouse-server/log.err.log</errorlog>
<size>1000M</size>
<count>10</count>
<stderr>/var/log/clickhouse-server/stderr.log</stderr>
<stdout>/var/log/clickhouse-server/stdout.log</stdout>
</logger>
<part_log>
<database>system</database>
<table>part_log</table>
<flush_interval_milliseconds>500</flush_interval_milliseconds>
</part_log>
</yandex>

Some files were not shown because too many files have changed in this diff Show More